Update infra_libs to 1.1.15 / 0b44aba87c1c6538439df6d24a409870810747ab

Hopefully (?) this should help reduce the amount of failures during instance initialization. [email protected] BUG=748707 Review-Url: https://codereview.chromium.org/2991803002
whesse · Jul 26, 2017 · 3737c64 · 3737c64
1 parent 37d8dd1
commit 3737c64
Show file tree

Hide file tree

Showing 43 changed files with 746 additions and 1,827 deletions.
diff --git a/appengine/swarming/server/bot_archive.py b/appengine/swarming/server/bot_archive.py
@@ -87,27 +87,13 @@
     'infra_libs/ts_mon/common/pb_to_popo.py',
     'infra_libs/ts_mon/common/standard_metrics.py',
     'infra_libs/ts_mon/common/targets.py',
-    'infra_libs/ts_mon/config.proto',
     'infra_libs/ts_mon/config.py',
     'infra_libs/ts_mon/protos/__init__.py',
-    'infra_libs/ts_mon/protos/current/__init__.py',
-    'infra_libs/ts_mon/protos/current/acquisition_network_device.proto',
-    'infra_libs/ts_mon/protos/current/acquisition_network_device_pb2.py',
-    'infra_libs/ts_mon/protos/current/acquisition_task.proto',
-    'infra_libs/ts_mon/protos/current/acquisition_task_pb2.py',
-    'infra_libs/ts_mon/protos/current/metrics.proto',
-    'infra_libs/ts_mon/protos/current/metrics_pb2.py',
-    'infra_libs/ts_mon/protos/new/__init__.py',
-    'infra_libs/ts_mon/protos/new/acquisition_network_device.proto',
-    'infra_libs/ts_mon/protos/new/acquisition_network_device_pb2.py',
-    'infra_libs/ts_mon/protos/new/acquisition_task.proto',
-    'infra_libs/ts_mon/protos/new/acquisition_task_pb2.py',
-    'infra_libs/ts_mon/protos/new/any.proto',
-    'infra_libs/ts_mon/protos/new/any_pb2.py',
-    'infra_libs/ts_mon/protos/new/metrics.proto',
-    'infra_libs/ts_mon/protos/new/metrics_pb2.py',
-    'infra_libs/ts_mon/protos/new/timestamp.proto',
-    'infra_libs/ts_mon/protos/new/timestamp_pb2.py',
+    'infra_libs/ts_mon/protos/acquisition_network_device_pb2.py',
+    'infra_libs/ts_mon/protos/acquisition_task_pb2.py',
+    'infra_libs/ts_mon/protos/any_pb2.py',
+    'infra_libs/ts_mon/protos/metrics_pb2.py',
+    'infra_libs/ts_mon/protos/timestamp_pb2.py',
     'infra_libs/utils.py',
     'libs/__init__.py',
     'libs/arfile/__init__.py',

diff --git a/appengine/third_party/gae_event_mon/README.swarming b/appengine/third_party/gae_event_mon/README.swarming
@@ -1,8 +1,8 @@
 Name: gae_event_mon
 Short Name: gae_event_mon
 URL: https://chromium.googlesource.com/infra/infra/+/master/appengine_module/gae_event_mon/
-Version: 1.1.1
-Revision: d9eec3590341f6cd4834ac49f1c662c0a0062d53
+Version: 1.1.15
+Revision: 0b44aba87c1c6538439df6d24a409870810747ab
 License: Chromium License
 
 Description:

diff --git a/appengine/third_party/gae_ts_mon/README.md b/appengine/third_party/gae_ts_mon/README.md
@@ -59,7 +59,7 @@
     something like `[email protected]`.
     Add it as a "Service Account Actor" of the "App Engine Metric Publishers"
     service account in the
-    (google.com:prodx-mon-chrome-infra project)[https://console.developers.google.com/iam-admin/serviceaccounts/project?project=google.com:prodx-mon-chrome-infra&organizationId=433637338589]
+    [google.com:prodx-mon-chrome-infra project](https://console.developers.google.com/iam-admin/serviceaccounts/project?project=google.com:prodx-mon-chrome-infra&organizationId=433637338589)
     by selecting it from the list and clicking "Permissions".
     If you see an error "You do not have viewing permissions for the selected
     resource.", then please ask the current chrome-trooper to do it for you.

diff --git a/appengine/third_party/gae_ts_mon/README.swarming b/appengine/third_party/gae_ts_mon/README.swarming
@@ -1,8 +1,8 @@
 Name: gae_ts_mon
 Short Name: gae_ts_mon
 URL: https://chromium.googlesource.com/infra/infra/+/master/appengine_module/gae_ts_mon/
-Version: 1.1.1
-Revision: d9eec3590341f6cd4834ac49f1c662c0a0062d53
+Version: 1.1.15
+Revision: 0b44aba87c1c6538439df6d24a409870810747ab
 License: Chromium License
 
 Description:

diff --git a/appengine/third_party/gae_ts_mon/__init__.py b/appengine/third_party/gae_ts_mon/__init__.py
@@ -56,6 +56,7 @@
 from infra_libs.ts_mon.common.errors import MonitoringNoConfiguredTargetError
 
 from infra_libs.ts_mon.common.helpers import ScopedIncrementCounter
+from infra_libs.ts_mon.common.helpers import ScopedMeasureTime
 
 from infra_libs.ts_mon.common.interface import close
 from infra_libs.ts_mon.common.interface import flush
@@ -66,16 +67,13 @@
 from infra_libs.ts_mon.common.metrics import IntegerField
 from infra_libs.ts_mon.common.metrics import StringField
 
-from infra_libs.ts_mon.common.metrics import BooleanField
-from infra_libs.ts_mon.common.metrics import IntegerField
-from infra_libs.ts_mon.common.metrics import StringField
-
 from infra_libs.ts_mon.common.metrics import BooleanMetric
 from infra_libs.ts_mon.common.metrics import CounterMetric
 from infra_libs.ts_mon.common.metrics import CumulativeDistributionMetric
 from infra_libs.ts_mon.common.metrics import CumulativeMetric
 from infra_libs.ts_mon.common.metrics import FloatMetric
 from infra_libs.ts_mon.common.metrics import GaugeMetric
+from infra_libs.ts_mon.common.metrics import MetricsDataUnits
 from infra_libs.ts_mon.common.metrics import NonCumulativeDistributionMetric
 from infra_libs.ts_mon.common.metrics import StringMetric
 

diff --git a/appengine/third_party/gae_ts_mon/config.py b/appengine/third_party/gae_ts_mon/config.py
@@ -80,7 +80,6 @@ def _flush_metrics(time_now):
     # ts_mon is not configured.
     return False
 
-  logging.info('ts_mon: fetching TSMonInstance entity')
   datetime_now = datetime.datetime.utcfromtimestamp(time_now)
   entity = shared.get_instance_entity()
   if entity.task_num < 0:
@@ -92,26 +91,18 @@ def _flush_metrics(time_now):
     if updated_sec_ago > shared.INSTANCE_EXPECTED_TO_HAVE_TASK_NUM_SEC:
       logging.warning('Instance %s is %d seconds old with no task_num.',
                       shared.instance_key_id(), updated_sec_ago)
-    logging.info('ts_mon: skipping the flush, no task_num')
     return False
   interface.state.target.task_num = entity.task_num
 
-  logging.info('ts_mon: initiating TSMonInstance entity update')
   entity.last_updated = datetime_now
   entity_deferred = entity.put_async()
 
-  logging.info('ts_mon: doing the flush')
-  started = time.time()
   interface.flush()
-  logging.info(
-      'ts_mon: flush finished in %d ms', (time.time() - started) * 1000)
 
   for metric in interface.state.global_metrics.itervalues():
     metric.reset()
 
-  logging.info('ts_mon: waiting for TSMonInstance entity update to finish')
   entity_deferred.get_result()
-  logging.info('ts_mon: flush is done')
   return True
 
 
@@ -218,11 +209,9 @@ def initialize(app=None, is_enabled_fn=None, cron_module='default',
 def _instrumented_dispatcher(dispatcher, request, response, time_fn=time.time):
   start_time = time_fn()
   response_status = 0
-  interface.state.store.initialize_context()
   flush_thread = None
   time_now = time_fn()
   if need_to_flush_metrics(time_now):
-    logging.info('ts_mon: starting flush thread')
     flush_thread = threading.Thread(target=_flush_metrics, args=(time_now,))
     flush_thread.start()
   try:
@@ -239,12 +228,7 @@ def _instrumented_dispatcher(dispatcher, request, response, time_fn=time.time):
     response_status = response.status_int
   finally:
     if flush_thread:
-      start_waiting = time_fn()
-      logging.info('ts_mon: joining flush thread')
       flush_thread.join()
-      logging.info(
-          'ts_mon: flush threads is done, waited %d ms',
-          (time_fn() - start_waiting) * 1000)
     elapsed_ms = int((time_fn() - start_time) * 1000)
 
     # Use the route template regex, not the request path, to prevent an
@@ -286,7 +270,6 @@ def decorated(service, *args, **kwargs):
       endpoint_name = '/_ah/spi/%s.%s' % (service_name, method_name)
       start_time = time_fn()
       response_status = 0
-      interface.state.store.initialize_context()
       flush_thread = None
       time_now = time_fn()
       if need_to_flush_metrics(time_now):
@@ -329,8 +312,6 @@ def _callable_name(self, fn):
     return '<unknown>'  # pragma: no cover
 
   def process_view(self, request, view_func, view_args, view_kwargs):
-    interface.state.store.initialize_context()
-
     time_now = self._time_fn()
     state = {
         'flush_thread': None,

diff --git a/client/third_party/infra_libs/README.swarming b/client/third_party/infra_libs/README.swarming
@@ -1,8 +1,8 @@
 Name: infra_libs
 Short Name: infra_libs
 URL: https://chromium.googlesource.com/infra/infra/+/master/packages/infra_libs/infra_libs/
-Version: 1.1.1
-Revision: d9eec3590341f6cd4834ac49f1c662c0a0062d53
+Version: 1.1.15
+Revision: 0b44aba87c1c6538439df6d24a409870810747ab
 License: Chromium License
 
 Description:

diff --git a/client/third_party/infra_libs/event_mon/config.py b/client/third_party/infra_libs/event_mon/config.py
@@ -11,7 +11,6 @@
   ChromeInfraEvent, ServiceEvent)
 from infra_libs.event_mon import router as ev_router
 
-DEFAULT_SERVICE_ACCOUNT_CREDS = 'service-account-event-mon.json'
 RUNTYPES = set(('dry', 'test', 'prod', 'file'))
 
 # Remote endpoints
@@ -37,33 +36,20 @@ def add_argparse_options(parser):
                      'do not do it.'
                      )
   group.add_argument('--event-mon-run-type', default='dry',
-                      choices=RUNTYPES,
-                      help='Determine how to send data. "dry" does not send\n'
-                      'anything. "test" sends to the test endpoint, \n'
-                      '"prod" to the actual production endpoint, and "file" \n'
-                      'writes to a file.')
+                     choices=RUNTYPES,
+                     help='Determine how to send data. "dry" does not send\n'
+                     'anything. "test" sends to the test endpoint, \n'
+                     '"prod" to the actual production endpoint, and "file" \n'
+                     'writes to a file.')
   group.add_argument('--event-mon-output-file', default='event_mon.output',
                      help='File into which LogEventLite serialized protos are\n'
                      'written when --event-mon-run-type is "file"')
   group.add_argument('--event-mon-service-name',
-                      help='Service name to use in log events.')
+                     help='Service name to use in log events.')
   group.add_argument('--event-mon-hostname',
-                      help='Hostname to use in log events.')
+                     help='Hostname to use in log events.')
   group.add_argument('--event-mon-appengine-name',
-                      help='App name to use in log events.')
-  group.add_argument('--event-mon-service-account-creds',
-                     default=DEFAULT_SERVICE_ACCOUNT_CREDS,
-                     metavar='JSON_FILE',
-                     help="Path to a json file containing a service account's"
-                     "\ncredentials. This is relative to the path specified\n"
-                     "in --event-mon-service-accounts-creds-root\n"
-                     "Defaults to '%(default)s'")
-  group.add_argument('--event-mon-service-accounts-creds-root',
-                     metavar='DIR',
-                     default=infra_libs.SERVICE_ACCOUNTS_CREDS_ROOT,
-                     help="Directory containing service accounts credentials.\n"
-                     "Defaults to %(default)s"
-                     )
+                     help='App name to use in log events.')
   group.add_argument('--event-mon-http-timeout', default=10, type=int,
                      help='Timeout in seconds for HTTP requests to send events')
   group.add_argument('--event-mon-http-retry-backoff', default=2., type=float,
@@ -83,8 +69,6 @@ def process_argparse_options(args):  # pragma: no cover
     hostname=args.event_mon_hostname,
     service_name=args.event_mon_service_name,
     appengine_name=args.event_mon_appengine_name,
-    service_account_creds=args.event_mon_service_account_creds,
-    service_accounts_creds_root=args.event_mon_service_accounts_creds_root,
     output_file=args.event_mon_output_file,
     dry_run=args.dry_run,
     http_timeout=args.event_mon_http_timeout,
@@ -96,8 +80,6 @@ def setup_monitoring(run_type='dry',
                      hostname=None,
                      service_name=None,
                      appengine_name=None,
-                     service_account_creds=None,
-                     service_accounts_creds_root=None,
                      output_file=None,
                      dry_run=False,
                      http_timeout=10,
@@ -122,13 +104,6 @@ def setup_monitoring(run_type='dry',
 
     appengine_name (str): name of the appengine app, if running on appengine.
 
-    service_account_creds (str): path to a json file containing a service
-      account's credentials obtained from a Google Cloud project. **Path is
-      relative to service_account_creds_root**, which is not the current path by
-      default. See infra_libs.authentication for details.
-
-    service_account_creds_root (str): path containing credentials files.
-
     output_file (str): file where to write the output in run_type == 'file'
       mode.
 
@@ -144,50 +119,46 @@ def setup_monitoring(run_type='dry',
   global _router
   logging.debug('event_mon: setting up monitoring.')
 
-  if not _router:
-    default_event = ChromeInfraEvent()
-
-    hostname = hostname or socket.getfqdn()
-    # hostname might be empty string or None on some systems, who knows.
-    if hostname:  # pragma: no branch
-      default_event.event_source.host_name = hostname
-    else:  # pragma: no cover
-      logging.warning('event_mon: unable to determine hostname.')
-
-    if service_name:
-      default_event.event_source.service_name = service_name
-    if appengine_name:
-      default_event.event_source.appengine_name = appengine_name
-
-    _cache['default_event'] = default_event
-    if run_type in ('prod', 'test'):
-      _cache['service_account_creds'] = service_account_creds
-      _cache['service_accounts_creds_root'] = service_accounts_creds_root
-    else:
-      _cache['service_account_creds'] = None
-      _cache['service_accounts_creds_root'] = None
-
-    if run_type not in RUNTYPES:
-      logging.error('Unknown run_type (%s). Setting to "dry"', run_type)
-      run_type = 'dry'
-
-    if run_type == 'dry':
-      # If we are running on AppEngine or devserver, use logging module.
-      server_software = os.environ.get('SERVER_SOFTWARE', '')
-      if (server_software.startswith('Google App Engine') or
-          server_software.startswith('Development')):
-        _router = ev_router._LoggingStreamRouter()
-      else:
-        _router = ev_router._TextStreamRouter()
-    elif run_type == 'file':
-      _router = ev_router._LocalFileRouter(output_file,
-                                           dry_run=dry_run)
+  if _router:
+    return
+
+  default_event = ChromeInfraEvent()
+
+  hostname = hostname or socket.getfqdn()
+  # hostname might be empty string or None on some systems, who knows.
+  if hostname:  # pragma: no branch
+    default_event.event_source.host_name = hostname
+  else:  # pragma: no cover
+    logging.warning('event_mon: unable to determine hostname.')
+
+  if service_name:
+    default_event.event_source.service_name = service_name
+  if appengine_name:
+    default_event.event_source.appengine_name = appengine_name
+
+  _cache['default_event'] = default_event
+
+  if run_type not in RUNTYPES:
+    logging.error('Unknown run_type (%s). Setting to "dry"', run_type)
+    run_type = 'dry'
+
+  if run_type == 'dry':
+    # If we are running on AppEngine or devserver, use logging module.
+    server_software = os.environ.get('SERVER_SOFTWARE', '')
+    if (server_software.startswith('Google App Engine') or
+        server_software.startswith('Development')):
+      _router = ev_router._LoggingStreamRouter()
     else:
-      _router = ev_router._HttpRouter(_cache,
-                                      ENDPOINTS.get(run_type),
-                                      dry_run=dry_run,
-                                      timeout=http_timeout,
-                                      retry_backoff=http_retry_backoff)
+      _router = ev_router._TextStreamRouter()
+  elif run_type == 'file':
+    _router = ev_router._LocalFileRouter(output_file,
+                                         dry_run=dry_run)
+  else:
+    _router = ev_router._HttpRouter(_cache,
+                                    ENDPOINTS.get(run_type),
+                                    dry_run=dry_run,
+                                    timeout=http_timeout,
+                                    retry_backoff=http_retry_backoff)
 
 
 def get_default_event():

diff --git a/client/third_party/infra_libs/event_mon/protos/chrome_infra_log_pb2.py b/client/third_party/infra_libs/event_mon/protos/chrome_infra_log_pb2.py