Документ взят из кэша поисковой машины. Адрес оригинального документа : http://www.stsci.edu/spst/UnixTransition/doc/met_util.py Дата изменения: Fri Apr 8 12:46:12 2016 Дата индексирования: Sun Apr 10 20:10:44 2016 Кодировка: Поисковые слова: images

#
#MODULE met_util
#
#*******************************************************************
"""

**PURPOSE** --

Collection of routines for metrics related actions

**DEVELOPER** --

Alan Patterson

**MODIFICATION HISTORY** --

Initial implementation 02/27/2002 APP.
Comments from Python review and updated
ftp information from CISD 05/03/2002 APP.
add code for efficiency metrics 08/01/2002 DC.
include both 50 line Excel and text file 08/19/2002 APP.
output.
Restore email to Ron and Mike for TofO 08/26/2002 APP.
weeks
Added datavolume_by_week distribution 12/02/2002 APP.
routine
Made distribution file lists generally 05/06/2003 APP.
accessible
Add TDRS & PASS process metric data 07/16/2003 APP.
Update datavolume routine 08/22/2003 APP.
Changes from python review 11/05/2003 APP
Prevent distribution if on OBS 04/19/2005 APP
Turn off the new spstftp option to
transfer to a temporary name 04/01/2014 DRC
Replace Alan's email with Joceyln's 04/29/2015 DRC
"""
#*******************************************************************
__author__ = "Alan Patterson"
__version__ = "15.04.29"

import os
import spss_sys_util
import transfer_util
import errno
import string
import time_util
import file_util

# METRIC_DIR is the local repository of metric files for SPST
METRIC_DIR = spss_sys_util.get_environ_variable("METRIC_DIR")[0]
# METRIC_TMP retains some files until distribution
METRIC_TMP = spss_sys_util.resolver("METRIC_DIR", 'tmp')

# Define tuples of Efficiency file names and transfer type as a dictionary
# on the file type
EFFICIENCY_LIST = [('efficiency_chart.txt', 'a'),
('efficiency_chart.xls', 'b'),
('efficiency_chart.ps', 'a')]

# Define tuples of TDRS and PASS metric file names and transfer type as a
# dictionary on the file type++
TDRS_PASS_LIST = [('tdrs_data.txt', 'a'),
('tdrs_data.xls', 'b'),
('dur_tdrs_ssaf.ps', 'a'),
('dur_tdrs_ssar.ps', 'a'),
('ms_process_stats.ps', 'a'),
('num_xmtr_on.ps', 'a')]

# Define tuples of Datavolume by week file names and transfer type as a
# dictionary on the file type
DATAVOLUME_BY_WEEK_LIST = [('datavolume_by_week.txt', 'a'),
('datavolume_by_week.xls', 'b'),
('datavolume_by_week.ps', 'a')]

# Define tuples of Datavolume file names and transfer type as a
# dictionary on the file type
DATAVOLUME_LIST = [('datavolume.txt', 'a'),
('datavolume.xls', 'b'),
('datavolume.ps', 'a')]

# METRIC_GROUPS contains the definitions of groups of metrics
# each dictionary entry is a list of dictionaries defining characteristics
# of publishable metrics:
METRIC_GROUPS = {'final_ms': ['datavolume', 'datavolume_by_week', 'tdrs_pass'],
'distrib_prod': ['efficiency']}


METRIC_DICT = {'datavolume': {'list': DATAVOLUME_LIST,
'src': METRIC_TMP, 'dst': 'DATAVOL_out',
'mail_string': 'datavolume'},
'datavolume_by_week': {'list': DATAVOLUME_BY_WEEK_LIST,
'src': METRIC_TMP,
'dst': 'DATAVOL_out',
'mail_string': 'datavolume by week'},
'efficiency': {'list': EFFICIENCY_LIST,
'src': METRIC_TMP, 'dst': 'EFF_out',
'mail_string': 'scheduling efficiency'},
'tdrs_pass': {'list': TDRS_PASS_LIST,
'src': METRIC_TMP, 'dst': 'TDRS_PASS_out',
'mail_string': 'TDRS & PASS process data'}}

# SENT_FILE logs the distribution of the individual metric representations
SENT_FILE = spss_sys_util.resolver("METRIC_DIR", 'sent_files.list')

# SUBDIRLIST contains the list of allowed subdirectories in the external
# metric area
SUBDIRLIST = ['DATAVOL_out', 'EFF_out', 'TDRS_PASS_out']

# email distribution lists
INTERNAL_EMAIL_ADDR = ['chance@stsci.edu', 'jferrara@stsci.edu']
DATAVOL_EMAIL_ADDR = ['downes@stsci.edu']


def send_metric_rep(filename, subdir, outfile, xfer_type):
"""Sends file to metrics subdirectory
where filename is the local file to be transferred
subdir is the subdirectory on the external metrics PC
(Example of subdir is : DATAVOL_out)
outfile is the output file name
"""
if (filename is None):
raise ValueError('no file specified!')

if not (os.path.exists(filename) and os.path.isfile(filename)):
# ENOENT is the standard file-not-found error
raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), filename)

if (subdir is None or subdir not in SUBDIRLIST):
raise ValueError(errno.EINVAL, 'Unrecognised subdirectory', subdir)

xtype = string.lower(xfer_type)
if (xtype not in ['a', 'b']):
raise ValueError(errno.EINVAL, 'Bad transfer type', xfer_type)

host = spss_sys_util.get_environ_variable('SPST_METRIC_FTP_NODE')[0]
user = spss_sys_util.get_environ_variable('SPST_METRIC_FTP_USER')[0]
base_dir = spss_sys_util.get_environ_variable('SPST_METRIC_FTP_DIR')[0]
word = '%s@stsci.edu' % os.environ["LOGNAME"]
ftp = transfer_util.spstftp(host, user, word)
ftp.cwd(base_dir)
if subdir is not None:
ftp.cwd(subdir)
result = ftp.put(filename, outfile, file_type=xtype, rename=False)
#a transfer initiated from VMS will cause certain errors to be reported
# 500 'STRU O VMS': command not understood.
# Structure VMS not supported by remote system.
# Transferred files may be corrupted.
#however the file may be readable if it was created under Unix
# Test with an Excel file were successful
ftp.quit()
if (result == '226 Transfer complete.'):
res = spss_sys_util.SUCCESS
else:
print result
res = not spss_sys_util.SUCCESS
return res


def distribute_metrics(metric_group, id):
"""Causes distribution of the specified metric group indicated by the
particular id.
"""
if (metric_group is None):
print distribute_metrics.__doc__
return not spss_sys_util.SUCCESS
if isinstance(metric_group, string):
if metric_group in METRIC_GROUPS:
for met in METRIC_GROUPS[metric_group]:
res = distrib_met(met)
else:
print 'Unrecognised metric name - %s' % metric_group
return not spss_sys_util.SUCCESS
elif isinstance(metric_group, list):
for met in metric_group:
res = distrib_met(met)
else:
print 'Unexpected arguement type'
return not spss_sys_util.SUCCESS

return res


def distrib_met(metric_name):
"""Distributes the metric representation files for the metric
'metric_name'.
"""

if metric_name not in METRIC_DICT:
print 'Unknown metric - %s' % metric_name
return not spss_sys_util.SUCCESS

# create list of files (metric representations) to send
# xfer_list will contain a list of 3 element tuples
# each tuple contains:
# 1. file path of metric representation to be sent
# 2. name file will have in the external metric area
# 3. file transfer type ('a' or 'b') i.e. Ascii or Binary
xfer_list = make_xfer_list(METRIC_DICT[metric_name]['list'],
METRIC_DICT[metric_name]['src'])

# transfer each file; add to email list if necessary
xferd_files = []
failed_files = []
for xfer in xfer_list:
try:
if spss_sys_util.on_obs():
print 'Bypassing sending %s' % xfer[1]
else:
send_metric_rep(xfer[0],
METRIC_DICT[metric_name]['dst'],
xfer[1],
xfer[2])
except:
print 'Metric file xfer failure - %s' % xfer[0]
failed_files.append(xfer[0])
else:
add_to_distrib_list(" ", " ", metric_name, xfer[1])
# append file to email list
xferd_files.append(xfer[1])

# email that the metric files have been distributed
if (len(xferd_files) != 0):
send_email(xferd_files,
METRIC_DICT[metric_name]['dst'],
METRIC_DICT[metric_name]['mail_string'],
INTERNAL_EMAIL_ADDR,
0)

# email if any metric files failed distribution
if (len(failed_files) != 0):
send_email(xferd_files,
METRIC_DICT[metric_name]['dst'],
METRIC_DICT[metric_name]['mail_string'],
INTERNAL_EMAIL_ADDR,
1)

return spss_sys_util.SUCCESS


def send_email(filelist, subdir, mstring, email_addr, fail_flag):
"""Sends alert email to list email_addr for files listed in xferd_files
and intended for the subdir subdirectory.
"""
tempfile = file_util.tempfile()
ftmp = open(tempfile, 'w')
if fail_flag == 0:
ftmp.write('The following files have just been transferred to the ')
ftmp.write('%s subdirectory\nfor the metrics for %s.\n\n' % (subdir, mstring))
for file in filelist:
ftmp.write('%s\n' % file)
subj = 'Metric files sent to %s' % subdir
else:
ftmp.write('The following files failed transfer to the ')
ftmp.write('%s subdirectory\nfor the metrics for %s.\n\n' % (subdir, mstring))
for file in filelist:
ftmp.write('%s\n' % file)
subj = 'Metric files failed distribution to %s' % subdir

ftmp.close()
for addr in email_addr:
spss_sys_util.mail(addr, subj, tempfile)
os.remove(tempfile)
return spss_sys_util.SUCCESS


def add_to_distrib_list(nom_week, ms_name, metric, outfile):
"""Appends file data to the list of already distributed products
for the nominal week
"""
fnom = open(SENT_FILE, 'a')
now = str(time_util.spss_time())
fnom.write('%s %s %s %s %s\n' % (now, nom_week, ms_name, metric, outfile))
fnom.close()
return spss_sys_util.SUCCESS


def make_xfer_list(xfer_file_list, input_dir):
"""Creates a list of 3 element tuples from the defined transfer list data
The input xfer_file_list is a list of 2 element tuples containing a
filename and transfer type
The output is a list with each element being a 3 element tuple:
1. file path of metric representation to be sent
2. name file will have in the external metric area
3. file transfer type ('a' or 'b') i.e. Ascii or Binary
"""
xlist = []
for fil in xfer_file_list:
xlist.append((os.path.join(input_dir, fil[0]),
spss_sys_util.basename(fil[0]),
fil[1]))
return xlist