Commits

Ville Suoranta authored 4787e80a5b2
Apply casalog changes from BB

casatelemetry/private/casatelemetry.py

Modified
1 1 import os
2 2 import fnmatch
3 3 import subprocess
4 4 import tarfile
5 5 import datetime
6 6 import time
7 7 import urllib.request
8 8 import urllib.error
9 9 import casatools
10 +from casatools import casalog
10 11 import casatasks
11 12 from casatelemetry import CrashReporter
12 13 import casatelemetry.private.TelemetryLogMonitor as TelemetryLogMonitor
13 14 import ssl
14 15 import logging
15 16 import sys
16 17
17 18 class telemetry:
18 19
19 20 def __init__(self, logdir_in=None):
20 21
21 22 from pathlib import Path
22 23 home = str(Path.home())
23 - self.casalogger = casatools.logsink()
24 24 self.setCasaVersion()
25 25 self.setHostId()
26 26 self.telemetry_enabled = True
27 27 casa_util = casatools.utils.utils()
28 28 if logdir_in == None:
29 29 self.logdir = home +"/.casa"
30 30 # Check if user has defined a telemetry log location
31 31 if (casatasks.config.telemetry_log_directory != None):
32 32 self.logdir = casatasks.config.telemetry_log_directory
33 33 else:
49 49 for file in os.listdir(self.logdir):
50 50 if fnmatch.fnmatch(file, self.logpattern):
51 51 #print ("Matched: " + file)
52 52 logfiles.append(file)
53 53
54 54 logfiles.sort(reverse=True)
55 55 # Size of the existing (non-active) logfiles
56 56 inactiveTLogSize = 0
57 57
58 58 if (logfiles and logfiles[0] != None):
59 - self.casalogger.post ("Found an existing telemetry logfile: " + self.logdir + "/" + logfiles[0])
59 + casalog.post ("Found an existing telemetry logfile: " + self.logdir + "/" + logfiles[0])
60 60 self.logfile= self.logdir + "/" + logfiles[0]
61 61 for i in range(1, len(logfiles)):
62 62 inactiveTLogSize = inactiveTLogSize + os.path.getsize(self.logdir + "/" + logfiles[i])/1024
63 63 #print ("Inactive log size: " + str(inactiveTLogSize))
64 64 self.init_log_file()
65 65 else :
66 66 print ("Creating a new telemetry file")
67 67 self.setNewTelemetryFile()
68 68
69 69
70 70
71 71 # Setup Telemetry log size monitoring
72 72 # Size limit for the telemetry logs
73 73 tLogSizeLimit = 20000
74 74 # File size check interval
75 75 tLogSizeInterval = 60
76 76
77 77 if (casatasks.config.telemetry_log_limit != None):
78 78 tLogSizeLimit = int(casatasks.config.telemetry_log_limit)
79 - self.casalogger.post("tLogSizeLimit: " + str(tLogSizeLimit))
79 + casalog.post("tLogSizeLimit: " + str(tLogSizeLimit))
80 80 if (casatasks.config.telemetry_log_size_interval != None):
81 81 tLogSizeInterval = int(casatasks.config.telemetry_log_size_interval)
82 - self.casalogger.post("tLogSizeInterval: " + str(tLogSizeInterval))
82 + casalog.post("tLogSizeInterval: " + str(tLogSizeInterval))
83 83
84 84 # Subtract the inactive log sizes from the total log file size limit
85 85 tLogSizeLimit = tLogSizeLimit - inactiveTLogSize
86 86 #print("tLogSizeLimit " + str(tLogSizeLimit))
87 87 if (tLogSizeLimit <= 0):
88 88 print ("Telemetry log size limit exceeded. Disabling telemetry.", file=sys.stderr)
89 89 self.telemetry_enabled = False
90 90 else :
91 91 tLogMonitor = TelemetryLogMonitor.TelemetryLogMonitor(ct_instance=self)
92 92 tLogMonitor.start(self.logfile,tLogSizeLimit, tLogSizeInterval)
98 98 def setNewTelemetryFile(self):
99 99 try:
100 100 self.logger.removehandler(self.loghandler)
101 101 except:
102 102 pass
103 103 self.logfile = self.logdir + '/casastats-' + self.casaver +'-' + self.hostid + "-" + time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + self.variantSuffix + '.log'
104 104 self.init_log_file()
105 105
106 106 def init_log_file(self):
107 107 try:
108 - self.casalogger.post("Telemetry log file: " + self.logfile)
108 + casalog.post("Telemetry log file: " + self.logfile)
109 109 self.logger = logging.getLogger('log')
110 110 self.logger.setLevel(logging.INFO)
111 111 self.loghandler = logging.FileHandler(self.logfile)
112 112 self.loghandler.setFormatter(logging.Formatter('%(message)s'))
113 113 self.logger.addHandler(self.loghandler)
114 114 except:
115 - self.casalogger.post("Telemetry log is not writeable.")
115 + casalog.post("Telemetry log is not writeable.")
116 116 pass
117 117
118 118 def setCasaVersion(self):
119 119 self.casaver = casatasks.version_string() # str(ver[0])+ str(ver[1]) + str(ver[2])+ "-" + str(ver[3])
120 120
121 121 def setHostId(self):
122 122 telemetryhelper = CrashReporter.CrashReportHelper()
123 123 self.hostid = telemetryhelper.getUniqueId()
124 124
125 125 #def setCasaLog(self, logger):
126 126 # self.logger = logger
127 127
128 128 def submitStatistics(self):
129 129 #if (self.casa['state']['telemetry-enabled'] == True):
130 - self.casalogger.post("Checking telemetry submission interval")
130 + casalog.post("Checking telemetry submission interval")
131 131 self.createStampFile()
132 132 if (self.isSubmitInterval()):
133 133 postingUrl = 'https://casa.nrao.edu/cgi-bin/crash-report.pl'
134 134 if 'CASA_CRASHREPORT_URL' in os.environ:
135 135 postingUrl = os.environ['CASA_CRASHREPORT_URL']
136 136 self.send(postingUrl)
137 137 self.refreshStampFile()
138 138 self.setNewTelemetryFile()
139 139
140 140
143 143 lastUpdateTime = time.time()
144 144 if (os.path.isfile(self.stampfile)):
145 145 lastUpdateTime = os.path.getmtime(self.stampfile)
146 146
147 147 # Check update checkSubmitInterval
148 148 interval = 604800
149 149 utils = casatools.utils.utils()
150 150 if (casatasks.config.telemetry_submit_interval != None):
151 151 interval = float(casatasks.config.telemetry_submit_interval)
152 152 if ((currentTime - lastUpdateTime)> interval):
153 - self.casalogger.post("Telemetry submit interval reached, submitting telemetry data.")
153 + casalog.post("Telemetry submit interval reached, submitting telemetry data.")
154 154 return True
155 155 else:
156 - self.casalogger.post("Telemetry submit interval not reached. Not submitting data.")
156 + casalog.post("Telemetry submit interval not reached. Not submitting data.")
157 157 #print "lastUpdateTime" +str(lastUpdateTime)
158 158 #print "currentTime" +str(currentTime)
159 - self.casalogger.post("Next telemetry data submission in: " + str(datetime.timedelta( \
159 + casalog.post("Next telemetry data submission in: " + str(datetime.timedelta( \
160 160 seconds=(interval-(currentTime-lastUpdateTime)))))
161 161 return False
162 162
163 163 def createStampFile(self):
164 164 #print "Checking for stampfile " + self.stampfile
165 165 if not os.path.isfile(self.stampfile):
166 - self.casalogger.post("Creating a new telemetry time stamp file." + self.stampfile)
166 + casalog.post("Creating a new telemetry time stamp file." + self.stampfile)
167 167 open(self.stampfile, 'a').close()
168 168
169 169 def refreshStampFile(self):
170 170 os.utime(self.stampfile, None)
171 171
172 172 def send(self, telemetry_url):
173 173
174 174 telemetryhelper = CrashReporter.CrashReportHelper()
175 175 logfiles = []
176 176
177 177 # Test if internet connection is available.
178 178 context = ssl._create_unverified_context()
179 179 try:
180 180 urllib.request.urlopen('https://casa.nrao.edu/', timeout=20, context=context)
181 181 except urllib.error.URLError as err:
182 - self.casalogger.post("No telemetry server available. Not submitting data")
182 + casalog.post("No telemetry server available. Not submitting data")
183 183 return
184 184
185 185 # Find logfiles
186 186 for file in os.listdir(self.logdir):
187 187 if fnmatch.fnmatch(file, self.sendlogpattern):
188 188 #print "Matched: " + file
189 189 logfiles.append(file)
190 190
191 191 if (len(logfiles) > 0):
192 192 #Tar logfiles
193 193 current_date = datetime.datetime.today().strftime('%Y%m%d%H%M%S')
194 194 tarfileid = self.logdir + "/telemetry-" \
195 195 + telemetryhelper.getUniqueId() + "-" \
196 196 + current_date + ".tar.gz"
197 197 try:
198 198 tar = tarfile.open(tarfileid, "w:gz")
199 199 for logfile in logfiles:
200 200 tar.add(self.logdir + "/" + logfile,
201 201 arcname='telemetry/'+logfile)
202 202 tar.close()
203 203 except Exception as e:
204 - self.casalogger.post("Couldn't create telemetry tarfile")
205 - self.casalogger.post(str(e))
204 + casalog.post("Couldn't create telemetry tarfile")
205 + casalog.post(str(e))
206 206
207 207 try:
208 208 file_param = 'file=@' + tarfileid #+ '\"'
209 209 # Submit tarfile
210 - #self.casalogger.post ['curl', '-F', file_param , telemetry_url]
210 + #casalog.post ['curl', '-F', file_param , telemetry_url]
211 211 proc = subprocess.Popen(['curl', '-F', file_param , telemetry_url],stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
212 212 cmd_out, cmd_err = proc.communicate()
213 213 if cmd_out != None:
214 - self.casalogger.post(cmd_out)
215 - self.casalogger.post(cmd_out, 'DEBUG1')
214 + casalog.post(cmd_out)
215 + casalog.post(cmd_out, 'DEBUG1')
216 216 if cmd_err != None:
217 - self.casalogger.post(cmd_err)
218 - self.casalogger.post(cmd_err, 'DEBUG1')
217 + casalog.post(cmd_err)
218 + casalog.post(cmd_err, 'DEBUG1')
219 219 except Exception as e:
220 - self.casalogger.post("Couldn't submit telemetry logs")
221 - self.casalogger.post(str(e))
220 + casalog.post("Couldn't submit telemetry logs")
221 + casalog.post(str(e))
222 222
223 223 # Remove files
224 224 for logfile in logfiles:
225 225 try:
226 226 os.remove(self.logdir + "/" + logfile)
227 227 except Exception as e:
228 - self.casalogger.post("Couldn't remove logfile " + self.logdir + "/" + logfile)
229 - self.casalogger.post(str(e))
228 + casalog.post("Couldn't remove logfile " + self.logdir + "/" + logfile)
229 + casalog.post(str(e))
230 230 #print "Removed " + self.logdir + "/" + logfile
231 231 try:
232 232 os.remove(tarfileid)
233 - self.casalogger.post("Removed" + tarfileid)
233 + casalog.post("Removed" + tarfileid)
234 234 except Exception as e:
235 - self.casalogger.post("Couldn't remove " + tarfileid)
236 - self.casalogger.post(str(e))
235 + casalog.post("Couldn't remove " + tarfileid)
236 + casalog.post(str(e))
237 237 else:
238 - self.casalogger.post("No telemetry files to submit.")
238 + casalog.post("No telemetry files to submit.")

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut