Add telemetry to track usage and errors (microsoft#2319)

## Add telemetry library and Olive-specific functionality
- Convert .NET OneCollector library to Python and add to
telemetry/library
- Track actions/errors with telemetry decorator
- Use heartbeat event to track general usage
- Add flag to disable telemetry, if desired (`--disable-telemetry`)
- Create telemetry cache so events can be sent when network connectivity
is convenient
Nits:
- Break out version into separate file
- Remove Python 3.9 from classifiers, since library only supports 3.10+
- Update minimum onnxscript version needed, since FOLDED_FROM_KEY
requires 0.5.3

Description of change for release notes:
> Add telemetry to help improve our products and services

## Checklist before requesting a review
- [ ] Add unit tests for this change.
- [x] Make sure all tests can pass.
- [x] Update documents if necessary.
- [x] Lint and apply fixes to your code by running `lintrunner -a`
- [x] Is this a user-facing change? If yes, give a description of this
change to be included in the release notes.

---------

Co-authored-by: shaahji <96227573+shaahji@users.noreply.github.com>

Author: bmehta001

README.md

@@ -114,6 +114,9 @@ The sample chat app to run is found as [model-chat.py](https://github.com/micros
 - [Documentation](https://microsoft.github.io/Olive)
 - [Recipes](https://github.com/microsoft/olive-recipes)
 
+## Data/Telemetry
+Distributions of this project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details.
+
 ## 🤝 Contributions and Feedback
 - We welcome contributions! Please read the [contribution guidelines](./CONTRIBUTING.md) for more details on how to contribute to the Olive project.
 - For feature requests or bug reports, file a [GitHub Issue](https://github.com/microsoft/Olive/issues).

docs/Privacy.md

@@ -0,0 +1,16 @@
+# Privacy
+
+## Data Collection
+The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the repository. There are also some features in the software that may enable Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft's privacy statement. Our privacy statement can be found [here](https://go.microsoft.com/fwlink/?LinkID=824704). You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices.
+
+***
+
+## Technical Details
+Olive uses the [OpenTelemetry](https://opentelemetry.io/) API for its implementation. Telemetry is turned ON by default. Based on user consent, this data may be periodically sent to Microsoft servers following GDPR and privacy regulations for anonymity and data access controls. Application, device, and version information is collected automatically.
+
+In addition, Olive may collect additional telemetry data such as:
+- Invoked commands
+- Performance data
+- Exception information
+
+Collection of this additional telemetry can be disabled by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. If telemetry is enabled, but cannot be sent to Microsoft, it will be stored locally and sent when a connection is available. You can override the default cache location by setting the `OLIVE_TELEMETRY_CACHE_PATH` environment variable to a valid file path.

olive/__init__.py

@@ -14,8 +14,6 @@
 _logger.addHandler(_sc)
 _logger.propagate = False
 
-__version__ = "0.11.0.dev0"
-
 # pylint: disable=C0413
 
 # Import Python API functions
@@ -33,10 +31,12 @@
     tune_session_params,
 )
 from olive.engine.output import ModelOutput, WorkflowOutput  # noqa: E402
+from olive.version import __version__  # noqa: E402
 
 __all__ = [
     "ModelOutput",
     "WorkflowOutput",
+    "__version__",
     # Python API functions
     "benchmark",
     "capture_onnx_graph",

olive/cli/auto_opt.py

@@ -14,6 +14,7 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     get_input_model_config,
     update_accelerator_options,
     update_shared_cache_options,
@@ -22,6 +23,7 @@
 from olive.constants import Precision
 from olive.hardware.constants import ExecutionProvider
 from olive.package_config import OlivePackageConfig
+from olive.telemetry import action
 
 
 class AutoOptCommand(BaseOliveCLICommand):
@@ -167,8 +169,10 @@ def register_subcommand(parser: ArgumentParser):
         add_shared_cache_options(sub_parser)
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=AutoOptCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/base.py

@@ -631,6 +631,12 @@ def add_search_options(sub_parser: ArgumentParser):
     search_strategy_group.add_argument("--seed", type=int, default=0, help="Random seed for search sampler")
 
 
+def add_telemetry_options(sub_parser: ArgumentParser):
+    """Add telemetry options to the sub_parser."""
+    sub_parser.add_argument("--disable_telemetry", action="store_true", help="Disable telemetry for this command.")
+    return sub_parser
+
+
 def update_search_options(args, config):
     to_replace = []
     to_replace.extend(

olive/cli/benchmark.py

@@ -11,10 +11,12 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     get_input_model_config,
     update_shared_cache_options,
 )
 from olive.common.utils import set_nested_dict_value
+from olive.telemetry import action
 
 
 class BenchmarkCommand(BaseOliveCLICommand):
@@ -69,8 +71,10 @@ def register_subcommand(parser: ArgumentParser):
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
         add_shared_cache_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=BenchmarkCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/capture_onnx.py

@@ -13,12 +13,14 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     get_diffusers_input_model,
     get_input_model_config,
     update_shared_cache_options,
 )
 from olive.common.utils import set_nested_dict_value
 from olive.model.utils.diffusers_utils import is_valid_diffusers_model
+from olive.telemetry import action
 
 
 class ModelBuilderAccuracyLevel(IntEnum):
@@ -170,8 +172,10 @@ def register_subcommand(parser: ArgumentParser):
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
         add_shared_cache_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=CaptureOnnxGraphCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/configure_qualcomm_sdk.py

@@ -4,7 +4,8 @@
 # --------------------------------------------------------------------------
 from argparse import ArgumentParser
 
-from olive.cli.base import BaseOliveCLICommand
+from olive.cli.base import BaseOliveCLICommand, add_telemetry_options
+from olive.telemetry import action
 
 
 class ConfigureQualcommSDKCommand(BaseOliveCLICommand):
@@ -21,9 +22,10 @@ def register_subcommand(parser: ArgumentParser):
             required=True,
             choices=["3.6", "3.8"],
         )
-
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=ConfigureQualcommSDKCommand)
 
+    @action
     def run(self):
         from olive.platform_sdk.qualcomm.configure.configure import configure
 

olive/cli/convert_adapters.py

@@ -6,8 +6,9 @@
 from argparse import ArgumentParser
 from typing import TYPE_CHECKING
 
-from olive.cli.base import BaseOliveCLICommand, add_logging_options
+from olive.cli.base import BaseOliveCLICommand, add_logging_options, add_telemetry_options
 from olive.common.utils import WeightsFileFormat, save_weights
+from olive.telemetry import action
 
 if TYPE_CHECKING:
     from numpy.typing import NDArray
@@ -75,8 +76,10 @@ def register_subcommand(parser: ArgumentParser):
             help="Quantization mode for int4 quantization of adapter weights. Default is symmetric.",
         )
         add_logging_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=ConvertAdaptersCommand)
 
+    @action
     def run(self):
         import torch
         from peft import LoraConfig, load_peft_weights

olive/cli/diffusion_lora.py

@@ -11,11 +11,13 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     update_shared_cache_options,
 )
 from olive.common.utils import set_nested_dict_value
 from olive.constants import DiffusersModelVariant
 from olive.passes.diffusers.lora import LRSchedulerType, MixedPrecision
+from olive.telemetry import action
 
 
 class DiffusionLoraCommand(BaseOliveCLICommand):
@@ -237,8 +239,10 @@ def register_subcommand(parser: ArgumentParser):
         add_shared_cache_options(sub_parser)
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=DiffusionLoraCommand)
 
+    @action
     def run(self):
         return self._run_workflow()