Commits
1 1 | import os |
2 2 | import fnmatch |
3 3 | import subprocess |
4 4 | import tarfile |
5 5 | import datetime |
6 6 | import time |
7 7 | import urllib.request |
8 8 | import urllib.error |
9 9 | import casatools |
10 + | from casatools import casalog |
10 11 | import casatasks |
11 12 | from casatelemetry import CrashReporter |
12 13 | import casatelemetry.private.TelemetryLogMonitor as TelemetryLogMonitor |
13 14 | import ssl |
14 15 | import logging |
15 16 | import sys |
16 17 | |
17 18 | class telemetry: |
18 19 | |
19 20 | def __init__(self, logdir_in=None): |
20 21 | |
21 22 | from pathlib import Path |
22 23 | home = str(Path.home()) |
23 - | self.casalogger = casatools.logsink() |
24 24 | self.setCasaVersion() |
25 25 | self.setHostId() |
26 26 | self.telemetry_enabled = True |
27 27 | casa_util = casatools.utils.utils() |
28 28 | if logdir_in == None: |
29 29 | self.logdir = home +"/.casa" |
30 30 | # Check if user has defined a telemetry log location |
31 31 | if (casatasks.config.telemetry_log_directory != None): |
32 32 | self.logdir = casatasks.config.telemetry_log_directory |
33 33 | else: |
49 49 | for file in os.listdir(self.logdir): |
50 50 | if fnmatch.fnmatch(file, self.logpattern): |
51 51 | #print ("Matched: " + file) |
52 52 | logfiles.append(file) |
53 53 | |
54 54 | logfiles.sort(reverse=True) |
55 55 | # Size of the existing (non-active) logfiles |
56 56 | inactiveTLogSize = 0 |
57 57 | |
58 58 | if (logfiles and logfiles[0] != None): |
59 - | self.casalogger.post ("Found an existing telemetry logfile: " + self.logdir + "/" + logfiles[0]) |
59 + | casalog.post ("Found an existing telemetry logfile: " + self.logdir + "/" + logfiles[0]) |
60 60 | self.logfile= self.logdir + "/" + logfiles[0] |
61 61 | for i in range(1, len(logfiles)): |
62 62 | inactiveTLogSize = inactiveTLogSize + os.path.getsize(self.logdir + "/" + logfiles[i])/1024 |
63 63 | #print ("Inactive log size: " + str(inactiveTLogSize)) |
64 64 | self.init_log_file() |
65 65 | else : |
66 66 | print ("Creating a new telemetry file") |
67 67 | self.setNewTelemetryFile() |
68 68 | |
69 69 | |
70 70 | |
71 71 | # Setup Telemetry log size monitoring |
72 72 | # Size limit for the telemetry logs |
73 73 | tLogSizeLimit = 20000 |
74 74 | # File size check interval |
75 75 | tLogSizeInterval = 60 |
76 76 | |
77 77 | if (casatasks.config.telemetry_log_limit != None): |
78 78 | tLogSizeLimit = int(casatasks.config.telemetry_log_limit) |
79 - | self.casalogger.post("tLogSizeLimit: " + str(tLogSizeLimit)) |
79 + | casalog.post("tLogSizeLimit: " + str(tLogSizeLimit)) |
80 80 | if (casatasks.config.telemetry_log_size_interval != None): |
81 81 | tLogSizeInterval = int(casatasks.config.telemetry_log_size_interval) |
82 - | self.casalogger.post("tLogSizeInterval: " + str(tLogSizeInterval)) |
82 + | casalog.post("tLogSizeInterval: " + str(tLogSizeInterval)) |
83 83 | |
84 84 | # Subtract the inactive log sizes from the total log file size limit |
85 85 | tLogSizeLimit = tLogSizeLimit - inactiveTLogSize |
86 86 | #print("tLogSizeLimit " + str(tLogSizeLimit)) |
87 87 | if (tLogSizeLimit <= 0): |
88 88 | print ("Telemetry log size limit exceeded. Disabling telemetry.", file=sys.stderr) |
89 89 | self.telemetry_enabled = False |
90 90 | else : |
91 91 | tLogMonitor = TelemetryLogMonitor.TelemetryLogMonitor(ct_instance=self) |
92 92 | tLogMonitor.start(self.logfile,tLogSizeLimit, tLogSizeInterval) |
98 98 | def setNewTelemetryFile(self): |
99 99 | try: |
100 100 | self.logger.removehandler(self.loghandler) |
101 101 | except: |
102 102 | pass |
103 103 | self.logfile = self.logdir + '/casastats-' + self.casaver +'-' + self.hostid + "-" + time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + self.variantSuffix + '.log' |
104 104 | self.init_log_file() |
105 105 | |
106 106 | def init_log_file(self): |
107 107 | try: |
108 - | self.casalogger.post("Telemetry log file: " + self.logfile) |
108 + | casalog.post("Telemetry log file: " + self.logfile) |
109 109 | self.logger = logging.getLogger('log') |
110 110 | self.logger.setLevel(logging.INFO) |
111 111 | self.loghandler = logging.FileHandler(self.logfile) |
112 112 | self.loghandler.setFormatter(logging.Formatter('%(message)s')) |
113 113 | self.logger.addHandler(self.loghandler) |
114 114 | except: |
115 - | self.casalogger.post("Telemetry log is not writeable.") |
115 + | casalog.post("Telemetry log is not writeable.") |
116 116 | pass |
117 117 | |
118 118 | def setCasaVersion(self): |
119 119 | self.casaver = casatasks.version_string() # str(ver[0])+ str(ver[1]) + str(ver[2])+ "-" + str(ver[3]) |
120 120 | |
121 121 | def setHostId(self): |
122 122 | telemetryhelper = CrashReporter.CrashReportHelper() |
123 123 | self.hostid = telemetryhelper.getUniqueId() |
124 124 | |
125 125 | #def setCasaLog(self, logger): |
126 126 | # self.logger = logger |
127 127 | |
128 128 | def submitStatistics(self): |
129 129 | #if (self.casa['state']['telemetry-enabled'] == True): |
130 - | self.casalogger.post("Checking telemetry submission interval") |
130 + | casalog.post("Checking telemetry submission interval") |
131 131 | self.createStampFile() |
132 132 | if (self.isSubmitInterval()): |
133 133 | postingUrl = 'https://casa.nrao.edu/cgi-bin/crash-report.pl' |
134 134 | if 'CASA_CRASHREPORT_URL' in os.environ: |
135 135 | postingUrl = os.environ['CASA_CRASHREPORT_URL'] |
136 136 | self.send(postingUrl) |
137 137 | self.refreshStampFile() |
138 138 | self.setNewTelemetryFile() |
139 139 | |
140 140 | |
143 143 | lastUpdateTime = time.time() |
144 144 | if (os.path.isfile(self.stampfile)): |
145 145 | lastUpdateTime = os.path.getmtime(self.stampfile) |
146 146 | |
147 147 | # Check update checkSubmitInterval |
148 148 | interval = 604800 |
149 149 | utils = casatools.utils.utils() |
150 150 | if (casatasks.config.telemetry_submit_interval != None): |
151 151 | interval = float(casatasks.config.telemetry_submit_interval) |
152 152 | if ((currentTime - lastUpdateTime)> interval): |
153 - | self.casalogger.post("Telemetry submit interval reached, submitting telemetry data.") |
153 + | casalog.post("Telemetry submit interval reached, submitting telemetry data.") |
154 154 | return True |
155 155 | else: |
156 - | self.casalogger.post("Telemetry submit interval not reached. Not submitting data.") |
156 + | casalog.post("Telemetry submit interval not reached. Not submitting data.") |
157 157 | #print "lastUpdateTime" +str(lastUpdateTime) |
158 158 | #print "currentTime" +str(currentTime) |
159 - | self.casalogger.post("Next telemetry data submission in: " + str(datetime.timedelta( \ |
159 + | casalog.post("Next telemetry data submission in: " + str(datetime.timedelta( \ |
160 160 | seconds=(interval-(currentTime-lastUpdateTime))))) |
161 161 | return False |
162 162 | |
163 163 | def createStampFile(self): |
164 164 | #print "Checking for stampfile " + self.stampfile |
165 165 | if not os.path.isfile(self.stampfile): |
166 - | self.casalogger.post("Creating a new telemetry time stamp file." + self.stampfile) |
166 + | casalog.post("Creating a new telemetry time stamp file." + self.stampfile) |
167 167 | open(self.stampfile, 'a').close() |
168 168 | |
169 169 | def refreshStampFile(self): |
170 170 | os.utime(self.stampfile, None) |
171 171 | |
172 172 | def send(self, telemetry_url): |
173 173 | |
174 174 | telemetryhelper = CrashReporter.CrashReportHelper() |
175 175 | logfiles = [] |
176 176 | |
177 177 | # Test if internet connection is available. |
178 178 | context = ssl._create_unverified_context() |
179 179 | try: |
180 180 | urllib.request.urlopen('https://casa.nrao.edu/', timeout=20, context=context) |
181 181 | except urllib.error.URLError as err: |
182 - | self.casalogger.post("No telemetry server available. Not submitting data") |
182 + | casalog.post("No telemetry server available. Not submitting data") |
183 183 | return |
184 184 | |
185 185 | # Find logfiles |
186 186 | for file in os.listdir(self.logdir): |
187 187 | if fnmatch.fnmatch(file, self.sendlogpattern): |
188 188 | #print "Matched: " + file |
189 189 | logfiles.append(file) |
190 190 | |
191 191 | if (len(logfiles) > 0): |
192 192 | #Tar logfiles |
193 193 | current_date = datetime.datetime.today().strftime('%Y%m%d%H%M%S') |
194 194 | tarfileid = self.logdir + "/telemetry-" \ |
195 195 | + telemetryhelper.getUniqueId() + "-" \ |
196 196 | + current_date + ".tar.gz" |
197 197 | try: |
198 198 | tar = tarfile.open(tarfileid, "w:gz") |
199 199 | for logfile in logfiles: |
200 200 | tar.add(self.logdir + "/" + logfile, |
201 201 | arcname='telemetry/'+logfile) |
202 202 | tar.close() |
203 203 | except Exception as e: |
204 - | self.casalogger.post("Couldn't create telemetry tarfile") |
205 - | self.casalogger.post(str(e)) |
204 + | casalog.post("Couldn't create telemetry tarfile") |
205 + | casalog.post(str(e)) |
206 206 | |
207 207 | try: |
208 208 | file_param = 'file=@' + tarfileid #+ '\"' |
209 209 | # Submit tarfile |
210 - | #self.casalogger.post ['curl', '-F', file_param , telemetry_url] |
210 + | #casalog.post ['curl', '-F', file_param , telemetry_url] |
211 211 | proc = subprocess.Popen(['curl', '-F', file_param , telemetry_url],stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
212 212 | cmd_out, cmd_err = proc.communicate() |
213 213 | if cmd_out != None: |
214 - | self.casalogger.post(cmd_out) |
215 - | self.casalogger.post(cmd_out, 'DEBUG1') |
214 + | casalog.post(cmd_out) |
215 + | casalog.post(cmd_out, 'DEBUG1') |
216 216 | if cmd_err != None: |
217 - | self.casalogger.post(cmd_err) |
218 - | self.casalogger.post(cmd_err, 'DEBUG1') |
217 + | casalog.post(cmd_err) |
218 + | casalog.post(cmd_err, 'DEBUG1') |
219 219 | except Exception as e: |
220 - | self.casalogger.post("Couldn't submit telemetry logs") |
221 - | self.casalogger.post(str(e)) |
220 + | casalog.post("Couldn't submit telemetry logs") |
221 + | casalog.post(str(e)) |
222 222 | |
223 223 | # Remove files |
224 224 | for logfile in logfiles: |
225 225 | try: |
226 226 | os.remove(self.logdir + "/" + logfile) |
227 227 | except Exception as e: |
228 - | self.casalogger.post("Couldn't remove logfile " + self.logdir + "/" + logfile) |
229 - | self.casalogger.post(str(e)) |
228 + | casalog.post("Couldn't remove logfile " + self.logdir + "/" + logfile) |
229 + | casalog.post(str(e)) |
230 230 | #print "Removed " + self.logdir + "/" + logfile |
231 231 | try: |
232 232 | os.remove(tarfileid) |
233 - | self.casalogger.post("Removed" + tarfileid) |
233 + | casalog.post("Removed" + tarfileid) |
234 234 | except Exception as e: |
235 - | self.casalogger.post("Couldn't remove " + tarfileid) |
236 - | self.casalogger.post(str(e)) |
235 + | casalog.post("Couldn't remove " + tarfileid) |
236 + | casalog.post(str(e)) |
237 237 | else: |
238 - | self.casalogger.post("No telemetry files to submit.") |
238 + | casalog.post("No telemetry files to submit.") |