Add profiling support to novaclient

boris-42 · Tovin Seven · commit 0fed79fd8fc8 · 2017-01-19T03:50:06.000Z
To be able to create profiling traces for Nova, client should be able to send special HTTP header that contains trace info. This patch is also important to be able to make cross project traces. (Typical case heat calls nova via python client, if profiler is initialized in heat, nova client will add extra header, that will be parsed by special osprofiler middleware in nova api.) Security considerations: trace information is signed by one of the HMAC keys that are set in nova.conf. So only person who knows HMAC key is able to send proper header. oslo-spec: https://review.openstack.org/#/c/103825/ Based on: https://review.openstack.org/#/c/105089/ Co-Authored-By: Dina Belova <dbelova@mirantis.com> Co-Authored-By: Roman Podoliaka <rpodolyaka@mirantis.com> Co-Authored-By: Tovin Seven <vinhnt@vn.fujitsu.com> Partially implements: blueprint osprofiler-support-in-nova Depends-On: I82d2badc8c1fcec27c3fce7c3c20e0f3b76414f1 Change-Id: I56ce4b547230e475854994c9d2249ef90e5b656c
diff --git a/novaclient/client.py b/novaclient/client.py
@@ -30,6 +30,9 @@
 from oslo_utils import importutils
 import pkg_resources
 
+osprofiler_profiler = importutils.try_import("osprofiler.profiler")
+osprofiler_web = importutils.try_import("osprofiler.web")
+
 from novaclient import api_versions
 from novaclient import exceptions
 from novaclient import extension as ext
@@ -67,6 +70,12 @@ def __init__(self, *args, **kwargs):
     def request(self, url, method, **kwargs):
         kwargs.setdefault('headers', kwargs.get('headers', {}))
         api_versions.update_headers(kwargs["headers"], self.api_version)
+
+        # NOTE(dbelova): osprofiler_web.get_trace_id_headers does not add any
+        # headers in case if osprofiler is not initialized.
+        if osprofiler_web:
+            kwargs['headers'].update(osprofiler_web.get_trace_id_headers())
+
         # NOTE(jamielennox): The standard call raises errors from
         # keystoneauth1, where we need to raise the novaclient errors.
         raise_exc = kwargs.pop('raise_exc', True)
@@ -343,6 +352,17 @@ def Client(version, username=None, password=None, project_id=None,
 
     api_version, client_class = _get_client_class_and_version(version)
     kwargs.pop("direct_use", None)
+
+    profile = kwargs.pop("profile", None)
+    if osprofiler_profiler and profile:
+        # Initialize the root of the future trace: the created trace ID will
+        # be used as the very first parent to which all related traces will be
+        # bound to. The given HMAC key must correspond to the one set in
+        # nova-api nova.conf, otherwise the latter will fail to check the
+        # request signature and will skip initialization of osprofiler on
+        # the server side.
+        osprofiler_profiler.init(profile)
+
     return client_class(api_version=api_version,
                         auth_url=auth_url,
                         direct_use=False,
diff --git a/novaclient/shell.py b/novaclient/shell.py
@@ -30,6 +30,8 @@
 from oslo_utils import strutils
 import six
 
+osprofiler_profiler = importutils.try_import("osprofiler.profiler")
+
 HAS_KEYRING = False
 all_errors = ValueError
 try:
@@ -509,6 +511,19 @@ def get_base_parser(self, argv):
             dest='endpoint_override',
             help=argparse.SUPPRESS)
 
+        if osprofiler_profiler:
+            parser.add_argument('--profile',
+                                metavar='HMAC_KEY',
+                                help='HMAC key to use for encrypting context '
+                                'data for performance profiling of operation. '
+                                'This key should be the value of the HMAC key '
+                                'configured for the OSprofiler middleware in '
+                                'nova; it is specified in the Nova '
+                                'configuration file at "/etc/nova/nova.conf". '
+                                'Without the key, profiling will not be '
+                                'triggered even if OSprofiler is enabled on '
+                                'the server side.')
+
         self._append_global_identity_args(parser, argv)
 
         return parser
@@ -749,6 +764,10 @@ def main(self, argv):
                 _("You must provide an auth url "
                   "via either --os-auth-url or env[OS_AUTH_URL]"))
 
+        additional_kwargs = {}
+        if osprofiler_profiler:
+            additional_kwargs["profile"] = args.profile
+
         # This client is just used to discover api version. Version API needn't
         # microversion, so we just pass version 2 at here.
         self.cs = client.Client(
@@ -767,7 +786,8 @@ def main(self, argv):
             project_domain_id=os_project_domain_id,
             project_domain_name=os_project_domain_name,
             user_domain_id=os_user_domain_id,
-            user_domain_name=os_user_domain_name)
+            user_domain_name=os_user_domain_name,
+            **additional_kwargs)
 
         if not skip_auth:
             if not api_version.is_latest():
@@ -858,6 +878,11 @@ def main(self, argv):
 
         args.func(self.cs, args)
 
+        if osprofiler_profiler and args.profile:
+            trace_id = osprofiler_profiler.get().get_base_id()
+            print("To display trace use the command:\n\n"
+                  "  osprofiler trace show --html %s " % trace_id)
+
         if args.timings:
             self._dump_timings(self.times + self.cs.get_timings())
 
diff --git a/novaclient/tests/unit/fixture_data/client.py b/novaclient/tests/unit/fixture_data/client.py
@@ -24,7 +24,8 @@
 class V1(fixtures.Fixture):
 
     def __init__(self, requests_mock,
-                 compute_url=COMPUTE_URL, identity_url=IDENTITY_URL):
+                 compute_url=COMPUTE_URL, identity_url=IDENTITY_URL,
+                 **client_kwargs):
         super(V1, self).__init__()
         self.identity_url = identity_url
         self.compute_url = compute_url
@@ -41,6 +42,8 @@ def __init__(self, requests_mock,
         s = self.token.add_service('computev3')
         s.add_endpoint(self.compute_url)
 
+        self._client_kwargs = client_kwargs
+
     def setUp(self):
         super(V1, self).setUp()
 
@@ -52,13 +55,14 @@ def setUp(self):
         self.requests_mock.get(self.identity_url,
                                json=self.discovery,
                                headers=headers)
-        self.client = self.new_client()
+        self.client = self.new_client(**self._client_kwargs)
 
-    def new_client(self):
+    def new_client(self, **client_kwargs):
         return client.Client("2", username='xx',
                              password='xx',
                              project_id='xx',
-                             auth_url=self.identity_url)
+                             auth_url=self.identity_url,
+                             **client_kwargs)
 
 
 class SessionV1(V1):
diff --git a/novaclient/tests/unit/test_shell.py b/novaclient/tests/unit/test_shell.py
@@ -655,6 +655,27 @@ def test_timing(self, m_times, m_requests):
         exc = self.assertRaises(RuntimeError, self.shell, '--timings list')
         self.assertEqual('Boom!', str(exc))
 
+    @requests_mock.Mocker()
+    def test_osprofiler(self, m_requests):
+        self.make_env()
+
+        def client(*args, **kwargs):
+            self.assertEqual('swordfish', kwargs['profile'])
+        with mock.patch('novaclient.client.Client', client):
+            # we are only interested in the fact Client is initialized properly
+            self.shell('list --profile swordfish', (0, 2))
+
+    @requests_mock.Mocker()
+    def test_osprofiler_not_installed(self, m_requests):
+        self.make_env()
+
+        # NOTE(rpodolyaka): osprofiler is in test-requirements, so we have to
+        # simulate its absence here
+        with mock.patch('novaclient.shell.osprofiler_profiler', None):
+            _, stderr = self.shell('list --profile swordfish', (0, 2))
+            self.assertIn('unrecognized arguments: --profile swordfish',
+                          stderr)
+
     @mock.patch('novaclient.shell.SecretsHelper.tenant_id',
                 return_value=True)
     @mock.patch('novaclient.shell.SecretsHelper.auth_token',
diff --git a/releasenotes/notes/add-osprofiler-support-cc9dd228242e9919.yaml b/releasenotes/notes/add-osprofiler-support-cc9dd228242e9919.yaml
@@ -0,0 +1,25 @@
+---
+prelude: >
+    OSprofiler support was added to the client. That makes
+    possible to trigger Nova operation trace generation from
+    the CLI.
+features:
+  - A new ``--profile`` option was added to allow Nova
+    profiling from the CLI. If the user wishes to trace a
+    nova boot request he or she needs to type the following
+    command -- ``nova --profile <secret_key> boot --image <image>
+    --flavor <flavor> <vm_name>``, where ``secret_key`` should match one
+    of the keys defined in nova.conf. As a result of this operation
+    additional information regarding ``trace_id`` will be
+    printed, that can be used to generate human-friendly
+    html report -- ``osprofiler trace show --html <trace_id> --out
+    trace.html``.
+    To enable profiling, user needs to have osprofiler
+    installed in the local environment via ``pip install osprofiler``.
+security:
+  - OSprofiler support, that was added during the Ocata release cycle,
+    requires passing of trace information between various
+    OpenStack services. This information is signed by one of
+    the HMAC keys defined in nova.conf file. That means that
+    only someone who knows this key is able to send the proper
+    header to trigger profiling.
diff --git a/test-requirements.txt b/test-requirements.txt
@@ -15,6 +15,7 @@ requests-mock>=1.1 # Apache-2.0
 sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
 os-client-config>=1.22.0 # Apache-2.0
 oslosphinx>=4.7.0 # Apache-2.0
+osprofiler>=1.4.0 # Apache-2.0
 testrepository>=0.0.18 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
