pax_global_header 0000666 0000000 0000000 00000000064 14172227630 0014516 g ustar 00root root 0000000 0000000 52 comment=62743e9125103d5cd636cccbd4dad6698cfcbb44
azure-kusto-python-3.0.1/ 0000775 0000000 0000000 00000000000 14172227630 0015327 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/.github/ 0000775 0000000 0000000 00000000000 14172227630 0016667 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/.github/ISSUE_TEMPLATE/ 0000775 0000000 0000000 00000000000 14172227630 0021052 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/.github/ISSUE_TEMPLATE/bug_report.md 0000664 0000000 0000000 00000001155 14172227630 0023546 0 ustar 00root root 0000000 0000000 ---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
#### Code Sample, a copy-pastable example if possible
```python
# Your code here
```
#### Problem description
[this should explain **why** the current behavior is a problem and why the expected output is a better solution.]
#### If query related, does it happen on other platforms (Kusto Web UI, Kusto Explorer)?
[this step is to help pin point problems that are only specific to this platform.]
#### Output of ``pip freeze``
[paste the output of ``pip freeze`` here below this line]
azure-kusto-python-3.0.1/.github/ISSUE_TEMPLATE/feature_request.md 0000664 0000000 0000000 00000001123 14172227630 0024574 0 ustar 00root root 0000000 0000000 ---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: ''
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.
azure-kusto-python-3.0.1/.github/pull_request_template.md 0000664 0000000 0000000 00000000564 14172227630 0023635 0 ustar 00root root 0000000 0000000 #### Pull Request Description
_[Add a description of your pull request here]_
---
#### Future Release Comment
_[Add description of your change, to include in the next release]_
_[Delete any or all irrelevant sections, e.g. if your change does not warrant a release comment at all]_
**Breaking Changes:**
- None
**Features:**
- None
**Fixes:**
- None azure-kusto-python-3.0.1/.github/workflows/ 0000775 0000000 0000000 00000000000 14172227630 0020724 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/.github/workflows/pythonpackage.yml 0000664 0000000 0000000 00000004042 14172227630 0024304 0 ustar 00root root 0000000 0000000 # This workflow will install the dependencies, run tests and lint every push
name: Python package
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: [ '3.7', '3.8', '3.9', '3.10' ]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r dev_requirements.txt
pip install codecov coverage
pip install ./azure-kusto-data ./azure-kusto-ingest
pip freeze
- uses: psf/black@stable
with:
options: "--check --diff --line-length 160"
- name: Test with pytest
run: |
pip install pytest
pytest -v --junit-xml pytest.xml
- name: EtoE Test with pytest
env:
APP_ID: ${{ secrets.APP_ID }}
APP_KEY: ${{ secrets.APP_KEY }}
AUTH_ID: ${{ secrets.AUTH_ID }}
TEST_DATABASE: ${{ secrets.TEST_DATABASE }}
TEST_BLOB: ${{ secrets.TEST_BLOB }}
ENGINE_CONNECTION_STRING: ${{ secrets.ENGINE_CONNECTION_STRING }}
run: |
pytest -v ./azure-kusto-ingest/tests/e2e.py --junit-xml pytest_e2e.xml
- name: Upload Unit Test Results
if: always()
uses: actions/upload-artifact@v2
with:
name: Unit Test Results (Python ${{ matrix.python-version }})
path: pytest*.xml
publish-test-results:
name: "Publish Unit Tests Results"
needs: build
runs-on: ubuntu-latest
if: always()
steps:
- name: Download Artifacts
uses: actions/download-artifact@v2
with:
path: artifacts
- name: Publish Unit Test Results
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: artifacts/**/*.xml azure-kusto-python-3.0.1/.github/workflows/pythonpublish.yml 0000664 0000000 0000000 00000002225 14172227630 0024360 0 ustar 00root root 0000000 0000000 # This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
name: Upload Python Package
on:
release:
types: [created]
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine
- name: Publish azure-kusto-data
working-directory: ./azure-kusto-data
env:
TWINE_USERNAME: microsoftkusto
TWINE_PASSWORD: ${{ secrets.pypi_password }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*
- name: Publish azure-kusto-ingest
working-directory: ./azure-kusto-ingest
env:
TWINE_USERNAME: microsoftkusto
TWINE_PASSWORD: ${{ secrets.pypi_password }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*
azure-kusto-python-3.0.1/.gitignore 0000664 0000000 0000000 00000011671 14172227630 0017325 0 ustar 00root root 0000000 0000000 ## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
##
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
dist/
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
**/Properties/launchSettings.json
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Typescript v1 declaration files
typings/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# CodeRush
.cr/
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# Result of running python setup.py install/pip install -e
RECORD.txt
build/
*.egg-info/
# Wheel files
*.whl
*.tar.gz
# pytest temp files
.pytest_cache*
# VS code files
.vscode*
#mypy files
.mypy*
*/.tox/ azure-kusto-python-3.0.1/.pylintrc 0000664 0000000 0000000 00000000132 14172227630 0017170 0 ustar 00root root 0000000 0000000 # https://github.com/getsentry/responses/issues/74
[TYPECHECK]
ignored-classes= responses
azure-kusto-python-3.0.1/CODE_OF_CONDUCT.md 0000664 0000000 0000000 00000000674 14172227630 0020135 0 ustar 00root root 0000000 0000000 # Microsoft Open Source Code of Conduct
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
Resources:
- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
azure-kusto-python-3.0.1/CONTRIBUTING.md 0000664 0000000 0000000 00000004266 14172227630 0017570 0 ustar 00root root 0000000 0000000 # Contributing to Azure Python SDK
If you would like to become an active contributor to this project please
follow the instructions provided in [Microsoft Azure Projects Contribution Guidelines](https://azure.github.io/azure-sdk/python_documentation.html).
## Requirements
In order to work on this project, we recommend using the dev requirements:
```bash
pip install -r dev_requirements.txt
```
These including testing related packages as well as styling ([black](https://black.readthedocs.io/en/stable/))
## Building and Testing
This project uses [pytest](https://docs.pytest.org/en/latest/).
Since the tests use the package as a third-party, the easiest way to set it up is installing it in edit mode:
```bash
pip install -e ./azure-kusto-data ./azure-kusto-ingest
```
After which, running tests is simple.
Just run:
```bash
pytest ./azure-kusto-data ./azure-kusto-ingest
```
## Style
We use black, and allow for line-length of 160, so please run:
```bash
black --line-length=160 ./azure-kusto-data ./azure-kusto-ingest
```
## PRs
We welcome contributions. In order to make the PR process efficient, please follow the below checklist:
* **There is an issue open concerning the code added** - (either bug or enhancement).
Preferably there is an agreed upon approach in the issue.
* **PR comment explains the changes done** - (This should be a TL;DR; as the rest of it should be documented in the related issue).
* **PR is concise** - try and avoid make drastic changes in a single PR. Split it into multiple changes if possible. If you feel a major change is needed, it is ok, but make sure commit history is clear and one of the maintainers can comfortably review both the code and the logic behind the change.
* **Please provide any related information needed to understand the change** - docs, guidelines, use-case, best practices and so on. Opinions are accepted, but have to be backed up.
* **Checks should pass** - these including linting with black and running tests.
## Code of Conduct
This project's code of conduct can be found in the
[CODE_OF_CONDUCT.md file](https://github.com/Azure/azure-sdk-for-python/blob/master/CODE_OF_CONDUCT.md)
(v1.4.0 of the http://contributor-covenant.org/ CoC).
azure-kusto-python-3.0.1/LICENSE 0000664 0000000 0000000 00000002212 14172227630 0016331 0 ustar 00root root 0000000 0000000 MIT License
Copyright (c) Microsoft Corporation. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE
azure-kusto-python-3.0.1/README.md 0000664 0000000 0000000 00000012405 14172227630 0016610 0 ustar 00root root 0000000 0000000 # Microsoft Azure Kusto (Azure Data Explorer) SDK for Python
[*azure-kusto-data*]("https://github.com/Azure/azure-kusto-python/tree/master/azure-kusto-data") Package provides the capability to query Kusto clusters with Python.
[](https://badge.fury.io/py/azure-kusto-data)
[](https://pepy.tech/project/azure-kusto-data)
[*azure-kusto-ingest*]("https://github.com/Azure/azure-kusto-python/tree/master/azure-kusto-ingest") Package allows sending data to Kusto service - i.e. ingest data.
[](https://badge.fury.io/py/azure-kusto-ingest)
[](https://pepy.tech/project/azure-kusto-ingest)
## Install
### Option 1: Via PyPi
To install via the Python Package Index (PyPI), type:
* `pip install azure-kusto-data`
* `pip install azure-kusto-ingest`
### Option 2: Source Via Git
To get the source code of the SDK via git just type:
```python
git clone git://github.com/Azure/azure-kusto-python.git
cd ./azure-kusto-python/azure-kusto-data
python setup.py install
cd ../azure-kusto-ingest
python setup.py install
```
### Option 3: Source Zip
Download a zip of the code via GitHub or PyPi. Then follow the same instructions in option 2.
### Optionals:
* [_Pandas_](http://pandas.pydata.org/) - Package provides extra functionality for use with pandas. Since these are optional dependencies, install with pandas:
* `pip install azure-kusto-data[pandas]`
* `pip install azure-kusto-ingest[pandas]`
## Minimum Requirements
* Python 3.5 and above
* See setup.py for dependencies
## Authentication methods:
* AAD Username/password - Provide your AAD username and password to Kusto client (**check the notice below**).
* AAD application - Provide app ID and app secret to Kusto client.
* AAD code - Provide only your AAD username, and authenticate yourself using a code, generated by ADAL.
* AZ CLI - For those already using [azure-cli](https://github.com/Azure/azure-cli), provide access token for the logged in user`.
** IMPORTANT NOTICE **:
User authentication (using username and password) has a major caveat:
Sometimes users are required to use Multi-Factor Authentication. In such a case, this flow won't work for them.
It is a limitation of the AAD library we are using under the hood. There are [several bugs reported](https://github.com/AzureAD/azure-activedirectory-library-for-python/issues?utf8=%E2%9C%93&q=is%3Aissue+mfa).
There is also a feature request for the adal team to work on implementing IWA (Intergrated Windows Auth) so that signed in users won't have to authenticate. Feel free to [upvote](https://github.com/AzureAD/microsoft-authentication-library-for-python/issues/31) if it is relevant in your case.
## Samples:
* [Kusto Quick Start Sample App](https://github.com/Azure/azure-kusto-python/tree/master/quick_start)
* [Kusto query sample sinppets](https://github.com/Azure/azure-kusto-python/blob/master/azure-kusto-data/tests/sample.py)
* [Data ingest sample sinppets](https://github.com/Azure/azure-kusto-python/blob/master/azure-kusto-ingest/tests/sample.py)
## Best Practices
See the SDK [best practices guide](https://docs.microsoft.com/azure/data-explorer/kusto/api/netfx/kusto-ingest-best-practices), which though written for the .NET SDK, applies similarly here.
## Need Support?
- **Have a feature request for SDKs?** Please post it on [User Voice](https://feedback.azure.com/forums/915733-azure-data-explorer) to help us prioritize
- **Have a technical question?** Ask on [Stack Overflow with tag "azure-data-explorer"](https://stackoverflow.com/questions/tagged/azure-data-explorer)
- **Need Support?** Every customer with an active Azure subscription has access to [support](https://docs.microsoft.com/en-us/azure/azure-supportability/how-to-create-azure-support-request) with guaranteed response time. Consider submitting a ticket and get assistance from Microsoft support team
- **Found a bug?** Please help us fix it by thoroughly documenting it and [filing an issue](https://github.com/Azure/azure-kusto-python/issues/new).
## Looking for SDKs for other languages/platforms?
- [Node](https://github.com/azure/azure-kusto-node)
- [Java](https://github.com/azure/azure-kusto-java)
- [.NET](https://docs.microsoft.com/en-us/azure/kusto/api/netfx/about-the-sdk)
- [Go](https://github.com/Azure/azure-kusto-go)
# Contribute
We gladly accept community contributions.
- Issues: Please report bugs using the Issues section of GitHub
- Forums: Interact with the development teams on StackOverflow or the Microsoft Azure Forums
- Source Code Contributions: If you would like to become an active contributor to this project please follow the instructions provided in [Contributing.md](CONTRIBUTING.md).
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
For general suggestions about Microsoft Azure please use our [UserVoice forum](http://feedback.azure.com/forums/34192--general-feedback).
azure-kusto-python-3.0.1/azure-kusto-data/ 0000775 0000000 0000000 00000000000 14172227630 0020527 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/MANIFEST.in 0000664 0000000 0000000 00000000047 14172227630 0022266 0 ustar 00root root 0000000 0000000 include *.rst
include azure/__init__.py azure-kusto-python-3.0.1/azure-kusto-data/README.rst 0000664 0000000 0000000 00000005360 14172227630 0022222 0 ustar 00root root 0000000 0000000 Microsoft Azure Kusto Library for Python
========================================
Overview
--------
.. code-block:: python
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
cluster = ""
client_id = ""
client_secret = ""
authority_id = ""
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, client_id, client_secret, authority_id)
client = KustoClient(kcsb)
db = "Samples"
query = "StormEvents | take 10"
response = client.execute(db, query)
for row in response.primary_results[0]:
print(row[0], " ", row["EventType"])
*Kusto Python Client* Library provides the capability to query Kusto clusters using Python.
It is Python 3.x compatible and supports
all data types through familiar Python DB API interface.
It's possible to use the library, for instance, from `Jupyter Notebooks
`_.
which are attached to Spark clusters,
including, but not exclusively, `Azure Databricks
`_. instances.
Async Client
~~~~~~~~~~~~
Kusto now provides an asynchronous client for queries.
To use the client, first install the package with the "aio" extra:
.. code:: bash
pip install azure-kusto-data[aio]
The async client uses exact same interface as the regular client, except
that it lives in the ``azure.kusto.data.aio`` namespace, and it returns
``Futures`` you will need to ``await`` its
.. code:: python
from azure.kusto.data import KustoConnectionStringBuilder
from azure.kusto.data.aio import KustoClient
cluster = ""
client_id = ""
client_secret = ""
authority_id = ""
async def sample():
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, client_id, client_secret, authority_id)
async with KustoClient(kcsb) as client:
db = "Samples"
query = "StormEvents | take 10"
response = await client.execute(db, query)
for row in response.primary_results[0]:
print(row[0], " ", row["EventType"])
Links
~~~~~
* `How to install the package `_.
* `Kusto query sample `_.
* `GitHub Repository `_.
azure-kusto-python-3.0.1/azure-kusto-data/azure/ 0000775 0000000 0000000 00000000000 14172227630 0021655 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/azure/__init__.py 0000664 0000000 0000000 00000000361 14172227630 0023766 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
"""Init file for azure namespace. https://github.com/Azure/azure-sdk-for-python/wiki/Azure-packaging"""
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/ 0000775 0000000 0000000 00000000000 14172227630 0023022 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/__init__.py 0000664 0000000 0000000 00000000361 14172227630 0025133 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
"""Init file for azure namespace. https://github.com/Azure/azure-sdk-for-python/wiki/Azure-packaging"""
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/ 0000775 0000000 0000000 00000000000 14172227630 0023733 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/__init__.py 0000664 0000000 0000000 00000000362 14172227630 0026045 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
from ._version import VERSION as __version__
from .client import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties
from .data_format import DataFormat
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/_cloud_settings.py 0000664 0000000 0000000 00000011111 14172227630 0027465 0 ustar 00root root 0000000 0000000 import os
from threading import Lock
from typing import Optional, Dict
from urllib.parse import urljoin
import requests
from azure.kusto.data.exceptions import KustoServiceError
METADATA_ENDPOINT = "v1/rest/auth/metadata"
DEFAULT_AUTH_ENV_VAR_NAME = "AadAuthorityUri"
DEFAULT_KUSTO_CLIENT_APP_ID = "db662dc1-0cfe-4e1c-a843-19a68e65be58"
DEFAULT_PUBLIC_LOGIN_URL = "https://login.microsoftonline.com"
DEFAULT_REDIRECT_URI = "https://microsoft/kustoclient"
DEFAULT_KUSTO_SERVICE_RESOURCE_ID = "https://kusto.kusto.windows.net"
DEFAULT_FIRST_PARTY_AUTHORITY_URL = "https://login.microsoftonline.com/f8cdef31-a31e-4b4a-93e4-5f571e91255a"
class CloudInfo:
"""This class holds the data for a specific cloud instance."""
def __init__(
self,
login_endpoint: str,
login_mfa_required: bool,
kusto_client_app_id: str,
kusto_client_redirect_uri: str,
kusto_service_resource_id: str,
first_party_authority_url: str,
):
self.login_endpoint = login_endpoint
self.login_mfa_required = login_mfa_required
self.kusto_client_app_id = kusto_client_app_id
self.kusto_client_redirect_uri = kusto_client_redirect_uri # will be used for interactive login
self.kusto_service_resource_id = kusto_service_resource_id
self.first_party_authority_url = first_party_authority_url
def authority_uri(self, authority_id: Optional[str]):
return self.login_endpoint + "/" + (authority_id or "organizations")
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return (
self.login_endpoint == other.login_endpoint
and self.login_mfa_required == other.login_mfa_required
and self.kusto_client_app_id == other.kusto_client_app_id
and self.kusto_client_redirect_uri == other.kusto_client_redirect_uri
and self.kusto_service_resource_id == other.kusto_service_resource_id
and self.first_party_authority_url == other.first_party_authority_url
)
class CloudSettings:
"""This class holds data for all cloud instances, and returns the specific data instance by parsing the dns suffix from a URL"""
_cloud_info = None
_cloud_cache = {}
_cloud_cache_lock = Lock()
DEFAULT_CLOUD = CloudInfo(
login_endpoint=os.environ.get(DEFAULT_AUTH_ENV_VAR_NAME, DEFAULT_PUBLIC_LOGIN_URL),
login_mfa_required=False,
kusto_client_app_id=DEFAULT_KUSTO_CLIENT_APP_ID,
kusto_client_redirect_uri=DEFAULT_REDIRECT_URI,
kusto_service_resource_id=DEFAULT_KUSTO_SERVICE_RESOURCE_ID,
first_party_authority_url=DEFAULT_FIRST_PARTY_AUTHORITY_URL,
)
@classmethod
def get_cloud_info_for_cluster(cls, kusto_uri: str, proxies: Optional[Dict[str, str]] = None) -> CloudInfo:
if kusto_uri in cls._cloud_cache: # Double-checked locking to avoid unnecessary lock access
return cls._cloud_cache[kusto_uri]
with cls._cloud_cache_lock:
if kusto_uri in cls._cloud_cache:
return cls._cloud_cache[kusto_uri]
result = requests.get(urljoin(kusto_uri, METADATA_ENDPOINT), proxies=proxies)
if result.status_code == 200:
content = result.json()
if content is None or content == {}:
raise KustoServiceError("Kusto returned an invalid cloud metadata response", result)
root = content["AzureAD"]
if root is not None:
cls._cloud_cache[kusto_uri] = CloudInfo(
login_endpoint=root["LoginEndpoint"],
login_mfa_required=root["LoginMfaRequired"],
kusto_client_app_id=root["KustoClientAppId"],
kusto_client_redirect_uri=root["KustoClientRedirectUri"],
kusto_service_resource_id=root["KustoServiceResourceId"],
first_party_authority_url=root["FirstPartyAuthorityUrl"],
)
else:
cls._cloud_cache[kusto_uri] = cls.DEFAULT_CLOUD
elif result.status_code == 404:
# For now as long not all proxies implement the metadata endpoint, if no endpoint exists return public cloud data
cls._cloud_cache[kusto_uri] = cls.DEFAULT_CLOUD
else:
raise KustoServiceError("Kusto returned an invalid cloud metadata response", result)
return cls._cloud_cache[kusto_uri]
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/_converters.py 0000664 0000000 0000000 00000001777 14172227630 0026652 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
import re
from datetime import timedelta
from dateutil import parser
# Regex for TimeSpan
_TIMESPAN_PATTERN = re.compile(r"(-?)((?P[0-9]*).)?(?P[0-9]{2}):(?P[0-9]{2}):(?P[0-9]{2}(\.[0-9]+)?$)")
def to_datetime(value):
"""Converts a string to a datetime."""
if isinstance(value, int):
return parser.parse(value)
return parser.isoparse(value)
def to_timedelta(value):
"""Converts a string to a timedelta."""
if isinstance(value, (int, float)):
return timedelta(microseconds=(float(value) / 10))
match = _TIMESPAN_PATTERN.match(value)
if match:
if match.group(1) == "-":
factor = -1
else:
factor = 1
return factor * timedelta(days=int(match.group("d") or 0), hours=int(match.group("h")), minutes=int(match.group("m")), seconds=float(match.group("s")))
else:
raise ValueError("Timespan value '{}' cannot be decoded".format(value))
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/_decorators.py 0000664 0000000 0000000 00000000516 14172227630 0026613 0 ustar 00root root 0000000 0000000 def aio_documented_by(original):
def wrapper(target):
target.__doc__ = "Aio function: {original_doc}".format(original_doc=original.__doc__)
return target
return wrapper
def documented_by(original):
def wrapper(target):
target.__doc__ = original.__doc__
return target
return wrapper
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/_models.py 0000664 0000000 0000000 00000016234 14172227630 0025735 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
import json
from abc import ABCMeta, abstractmethod
from decimal import Decimal
from enum import Enum
from typing import Iterator, List, Any, Union, Optional, Dict
from . import _converters
from .exceptions import KustoMultiApiError, KustoStreamingQueryError
class WellKnownDataSet(Enum):
"""Categorizes data tables according to the role they play in the data set that a Kusto query returns."""
PrimaryResult = "PrimaryResult"
QueryCompletionInformation = "QueryCompletionInformation"
TableOfContents = "TableOfContents"
QueryProperties = "QueryProperties"
class KustoResultRow:
"""Iterator over a Kusto result row."""
conversion_funcs = {"datetime": _converters.to_datetime, "timespan": _converters.to_timedelta, "decimal": Decimal}
def __init__(self, columns: "List[KustoResultColumn]", row: list):
self._value_by_name = {}
self._value_by_index = []
for i, value in enumerate(row):
column = columns[i]
try:
column_type = column.column_type.lower()
except AttributeError:
self._value_by_index.append(value)
self._value_by_name[columns[i]] = value
continue
# If you are here to read this, you probably hit some datetime/timedelta inconsistencies.
# Azure-Data-Explorer(Kusto) supports 7 decimal digits, while the corresponding python types supports only 6.
# One example why one might want this precision, is when working with pandas.
# In that case, use azure.kusto.data.helpers.dataframe_from_result_table which takes into account the original value.
typed_value = self.get_typed_value(column_type, value)
self._value_by_index.append(typed_value)
self._value_by_name[column.column_name] = typed_value
@staticmethod
def get_typed_value(column_type: str, value: Any) -> Any:
return KustoResultRow.conversion_funcs[column_type](value) if value is not None and column_type in KustoResultRow.conversion_funcs else value
@property
def columns_count(self) -> int:
return len(self._value_by_name)
def __iter__(self) -> Iterator[Any]:
for i in range(self.columns_count):
yield self[i]
def __getitem__(self, key: Union[str, int]) -> Any:
if isinstance(key, int):
return self._value_by_index[key]
return self._value_by_name[key]
def __len__(self) -> int:
return self.columns_count
def to_dict(self) -> Dict[str, Any]:
return self._value_by_name
def to_list(self) -> list:
return self._value_by_index
def __str__(self) -> str:
return "['{}']".format("', '".join([str(val) for val in self._value_by_index]))
def __repr__(self) -> str:
values = [repr(val) for val in self._value_by_name.values()]
return "KustoResultRow(['{}'], [{}])".format("', '".join(self._value_by_name), ", ".join(values))
def __eq__(self, other) -> bool:
if len(self) != len(other):
return False
for value_index, value in enumerate(self):
if value != other[value_index]:
return False
return True
class KustoResultColumn:
def __init__(self, json_column: Dict[str, Any], ordinal: int):
self.column_name = json_column["ColumnName"]
self.column_type = json_column.get("ColumnType") or json_column["DataType"]
self.ordinal = ordinal
def __repr__(self) -> str:
return "KustoResultColumn({},{})".format(json.dumps({"ColumnName": self.column_name, "ColumnType": self.column_type}), self.ordinal)
class BaseKustoResultTable(metaclass=ABCMeta):
def __init__(self, json_table: Dict[str, Any]):
self.table_name = json_table.get("TableName")
self.table_id = json_table.get("TableId")
self.table_kind = WellKnownDataSet[json_table["TableKind"]] if "TableKind" in json_table else None
self.columns = [KustoResultColumn(column, index) for index, column in enumerate(json_table["Columns"])]
self.raw_columns = json_table["Columns"]
self.raw_rows = json_table["Rows"]
self.kusto_result_rows = None
def __bool__(self) -> bool:
return any(self.columns)
__nonzero__ = __bool__
@property
def columns_count(self) -> int:
return len(self.columns)
@abstractmethod
def __len__(self) -> Optional[int]:
pass
@property
@abstractmethod
def rows_count(self) -> int:
pass
class BaseStreamingKustoResultTable(BaseKustoResultTable):
def __init__(self, json_table: Dict[str, Any]):
super().__init__(json_table)
self.finished = False
self.row_count = 0
@property
def rows_count(self) -> int:
if not self.finished:
raise KustoStreamingQueryError("Can't retrieve rows count before the iteration is finished")
return self.row_count
def __len__(self) -> Optional[int]:
if not self.finished:
return None # We return None here instead of an exception, because otherwise calling list() on the object will throw
return self.rows_count
def iter_rows(self) -> "BaseStreamingKustoResultTable":
return self
class KustoResultTable(BaseKustoResultTable):
"""Iterator over a Kusto result table."""
def __init__(self, json_table: Dict[str, Any]):
super().__init__(json_table)
errors = [row for row in json_table["Rows"] if isinstance(row, dict)]
if errors:
raise KustoMultiApiError(errors)
@property
def rows(self) -> List[KustoResultRow]:
if not self.kusto_result_rows:
self.kusto_result_rows = [KustoResultRow(self.columns, row) for row in self.raw_rows]
return self.kusto_result_rows
def to_dict(self) -> Dict[str, Any]:
"""Converts the table to a dict."""
return {"name": self.table_name, "kind": self.table_kind, "data": [r.to_dict() for r in self]}
@property
def rows_count(self) -> int:
return len(self.raw_rows)
def __len__(self) -> int:
return self.rows_count
def __iter__(self) -> Iterator[KustoResultRow]:
for row_index, row in enumerate(self.raw_rows):
if self.kusto_result_rows:
yield self.kusto_result_rows[row_index]
else:
yield KustoResultRow(self.columns, row)
def __getitem__(self, key: int) -> KustoResultRow:
return self.rows[key]
def __str__(self) -> str:
d = self.to_dict()
# enum is not serializable, using value instead
d["kind"] = d["kind"].value
return json.dumps(d, default=str)
class KustoStreamingResultTable(BaseStreamingKustoResultTable):
"""
Iterator over a Kusto result table in streaming.
This class can be iterated only once.
"""
def __next__(self) -> KustoResultRow:
try:
row = next(self.raw_rows)
except StopIteration:
self.finished = True
raise
self.row_count += 1
return KustoResultRow(self.columns, row)
def __iter__(self) -> Iterator[KustoResultRow]:
return self
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/_token_providers.py 0000664 0000000 0000000 00000055156 14172227630 0027675 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import abc
import asyncio
import time
import webbrowser
from threading import Lock
from typing import Callable, Optional, Coroutine, List
from azure.core.exceptions import ClientAuthenticationError
from azure.identity import ManagedIdentityCredential, AzureCliCredential
from msal import ConfidentialClientApplication, PublicClientApplication
from ._cloud_settings import CloudSettings, CloudInfo
from .exceptions import KustoClientError, KustoAioSyntaxError, KustoAsyncUsageError
try:
from asgiref.sync import sync_to_async
except ImportError:
def sync_to_async(f):
raise KustoAioSyntaxError()
try:
from azure.identity.aio import ManagedIdentityCredential as AsyncManagedIdentityCredential, AzureCliCredential as AsyncAzureCliCredential
except ImportError:
# These are here in case the user doesn't have the aio optional dependency installed, but still tries to use async.
# They will give them a useful error message, and will appease linters.
class AsyncManagedIdentityCredential:
def __init__(self):
raise KustoAioSyntaxError()
class AsyncAzureCliCredential:
def __init__(self):
raise KustoAioSyntaxError()
# constant key names and values used throughout the code
class TokenConstants:
BEARER_TYPE = "Bearer"
MSAL_TOKEN_TYPE = "token_type"
MSAL_ACCESS_TOKEN = "access_token"
MSAL_ERROR = "error"
MSAL_ERROR_DESCRIPTION = "error_description"
MSAL_PRIVATE_CERT = "private_key"
MSAL_THUMBPRINT = "thumbprint"
MSAL_PUBLIC_CERT = "public_certificate"
MSAL_DEVICE_MSG = "message"
MSAL_DEVICE_URI = "verification_uri"
MSAL_INTERACTIVE_PROMPT = "select_account"
AZ_TOKEN_TYPE = "tokenType"
AZ_ACCESS_TOKEN = "accessToken"
class TokenProviderBase(abc.ABC):
"""
This base class abstracts token acquisition for all implementations.
The class is build for Lazy initialization, so that the first call, take on instantiation of 'heavy' long-lived class members
"""
_initialized: bool = False
_resources_initialized: bool = False
def __init__(self, is_async: bool = False):
self._proxy_dict: Optional[str, str] = None
self.is_async = is_async
if is_async:
self._async_lock = asyncio.Lock()
else:
self._lock = Lock()
def _init_once(self, init_only_resources=False):
if self._initialized:
return
with self._lock:
if self._initialized:
return
if not self._resources_initialized:
self._init_resources()
self._resources_initialized = True
if init_only_resources:
return
self._init_impl()
self._initialized = True
async def _init_once_async(self, init_only_resources=False):
if self._initialized:
return
async with self._async_lock:
if self._initialized:
return
if not self._resources_initialized:
await (sync_to_async(self._init_resources)())
self._resources_initialized = True
if init_only_resources:
return
self._init_impl()
self._initialized = True
def _init_resources(self):
pass
def get_token(self):
"""Get a token silently from cache or authenticate if cached token is not found"""
if self.is_async:
raise KustoAsyncUsageError("get_token", self.is_async)
self._init_once()
token = self._get_token_from_cache_impl()
if token is None:
with self._lock:
token = self._get_token_impl()
return self._valid_token_or_throw(token)
def context(self) -> dict:
if self.is_async:
raise KustoAsyncUsageError("context", self.is_async)
self._init_once(init_only_resources=True)
return self._context_impl()
async def context_async(self) -> dict:
if not self.is_async:
raise KustoAsyncUsageError("context_async", self.is_async)
await self._init_once_async(init_only_resources=True)
return self._context_impl()
async def get_token_async(self):
"""Get a token asynchronously silently from cache or authenticate if cached token is not found"""
if not self.is_async:
raise KustoAsyncUsageError("get_token_async", self.is_async)
await self._init_once_async()
token = self._get_token_from_cache_impl()
if token is None:
async with self._async_lock:
token = await self._get_token_impl_async()
return self._valid_token_or_throw(token)
@staticmethod
@abc.abstractmethod
def name() -> str:
"""return the provider class name"""
pass
@abc.abstractmethod
def _context_impl(self) -> dict:
"""return a secret-free context for error reporting"""
pass
@abc.abstractmethod
def _init_impl(self):
"""Implement any "heavy" first time initializations here"""
pass
@abc.abstractmethod
def _get_token_impl(self) -> Optional[dict]:
"""implement actual token acquisition here"""
pass
async def _get_token_impl_async(self) -> Optional[dict]:
"""implement actual token acquisition here"""
return await sync_to_async(self._get_token_impl)()
@abc.abstractmethod
def _get_token_from_cache_impl(self) -> Optional[dict]:
"""Implement cache checks here, return None if cache check fails"""
pass
@staticmethod
def _valid_token_or_none(token: dict) -> Optional[dict]:
if token is None or TokenConstants.MSAL_ERROR in token:
return None
return token
def _valid_token_or_throw(self, token: dict, context: str = "") -> dict:
if token is None:
raise KustoClientError(self.name() + " - failed to obtain a token. " + context)
if TokenConstants.MSAL_ERROR in token:
message = self.name() + " - failed to obtain a token. " + context + "\n" + token[TokenConstants.MSAL_ERROR]
if TokenConstants.MSAL_ERROR_DESCRIPTION in token:
message = message + "\n" + token[TokenConstants.MSAL_ERROR_DESCRIPTION]
raise KustoClientError(message)
return token
def set_proxy(self, proxy_url: str):
self._proxy_dict = {"http": proxy_url, "https": proxy_url}
class CloudInfoTokenProvider(TokenProviderBase, abc.ABC):
_cloud_info: Optional[CloudInfo]
_scopes = List[str]
_kusto_uri: str
def __init__(self, kusto_uri: str, is_async: bool = False):
super().__init__(is_async)
self._kusto_uri = kusto_uri
def _init_resources(self):
if self._kusto_uri is not None:
self._cloud_info = CloudSettings.get_cloud_info_for_cluster(self._kusto_uri, self._proxy_dict)
resource_uri = self._cloud_info.kusto_service_resource_id
if self._cloud_info.login_mfa_required:
resource_uri = resource_uri.replace(".kusto.", ".kustomfa.")
self._scopes = [resource_uri + "/.default"]
class BasicTokenProvider(TokenProviderBase):
"""Basic Token Provider keeps and returns a token received on construction"""
def __init__(self, token: str, is_async: bool = False):
super().__init__(is_async)
self._token = token
@staticmethod
def name() -> str:
return "BasicTokenProvider"
def _context_impl(self) -> dict:
return {"authority": self.name()}
def _init_impl(self):
pass
def _get_token_impl(self) -> Optional[dict]:
return None
def _get_token_from_cache_impl(self) -> dict:
return {TokenConstants.MSAL_TOKEN_TYPE: TokenConstants.BEARER_TYPE, TokenConstants.MSAL_ACCESS_TOKEN: self._token}
class CallbackTokenProvider(TokenProviderBase):
"""Callback Token Provider generates a token based on a callback function provided by the caller"""
def __init__(
self, token_callback: Optional[Callable[[], str]], async_token_callback: Optional[Callable[[], Coroutine[None, None, str]]], is_async: bool = False
):
super().__init__(is_async)
self._token_callback = token_callback
self._async_token_callback = async_token_callback
@staticmethod
def name() -> str:
return "CallbackTokenProvider"
def _context_impl(self) -> dict:
return {"authority": self.name()}
def _init_impl(self):
pass
@staticmethod
def _build_response(caller_token) -> dict:
if not isinstance(caller_token, str):
raise KustoClientError("Token provider returned something that is not a string [" + str(type(caller_token)) + "]")
return {TokenConstants.MSAL_TOKEN_TYPE: TokenConstants.BEARER_TYPE, TokenConstants.MSAL_ACCESS_TOKEN: caller_token}
def _get_token_impl(self) -> Optional[dict]:
if self._token_callback is None:
raise KustoClientError("token_callback is None, can't retrieve token")
return self._build_response(self._token_callback())
async def _get_token_impl_async(self) -> Optional[dict]:
if self._async_token_callback is None:
return await super()._get_token_impl_async()
return self._build_response(await self._async_token_callback())
def _get_token_from_cache_impl(self) -> Optional[dict]:
return None
class MsiTokenProvider(CloudInfoTokenProvider):
"""
MSI Token Provider obtains a token from the MSI endpoint
The args parameter is a dictionary conforming with the ManagedIdentityCredential initializer API arguments
"""
def __init__(self, kusto_uri: str, msi_args: dict = None, is_async: bool = False):
super().__init__(kusto_uri, is_async)
self._msi_args = msi_args
self._msi_auth_context = None
self._msi_auth_context_async = None
@staticmethod
def name() -> str:
return "MsiTokenProvider"
def _context_impl(self) -> dict:
context = self._msi_args.copy()
context["authority"] = self.name()
return context
def _init_impl(self):
pass
def _get_token_impl(self) -> Optional[dict]:
try:
if self._msi_auth_context is None:
self._msi_auth_context = ManagedIdentityCredential(**self._msi_args)
msi_token = self._msi_auth_context.get_token(self._scopes[0])
return {TokenConstants.MSAL_TOKEN_TYPE: TokenConstants.BEARER_TYPE, TokenConstants.MSAL_ACCESS_TOKEN: msi_token.token}
except ClientAuthenticationError as e:
raise KustoClientError("Failed to initialize MSI ManagedIdentityCredential with [{0}]\n{1}".format(self._msi_args, e))
except Exception as e:
raise KustoClientError("Failed to obtain MSI token for '{0}' with [{1}]\n{2}".format(self._kusto_uri, self._msi_args, e))
async def _get_token_impl_async(self) -> Optional[dict]:
try:
if self._msi_auth_context_async is None:
self._msi_auth_context_async = AsyncManagedIdentityCredential(**self._msi_args)
msi_token = await self._msi_auth_context_async.get_token(self._scopes[0])
return {TokenConstants.MSAL_TOKEN_TYPE: TokenConstants.BEARER_TYPE, TokenConstants.MSAL_ACCESS_TOKEN: msi_token.token}
except ClientAuthenticationError as e:
raise KustoClientError("Failed to initialize MSI async ManagedIdentityCredential with [{0}]\n{1}".format(self._msi_args, e))
except Exception as e:
raise KustoClientError("Failed to obtain MSI token for '{0}' with [{1}]\n{2}".format(self._kusto_uri, self._msi_args, e))
def _get_token_from_cache_impl(self) -> Optional[dict]:
return None
class AzCliTokenProvider(CloudInfoTokenProvider):
"""AzCli Token Provider obtains a refresh token from the AzCli cache and uses it to authenticate with MSAL"""
def __init__(self, kusto_uri: str, is_async: bool = False):
super().__init__(kusto_uri, is_async)
self._az_auth_context = None
self._az_auth_context_async = None
self._az_token = None
@staticmethod
def name() -> str:
return "AzCliTokenProvider"
def _context_impl(self) -> dict:
return {"authority:": self.name()}
def _init_impl(self):
pass
def _get_token_impl(self) -> Optional[dict]:
try:
if self._az_auth_context is None:
self._az_auth_context = AzureCliCredential()
self._az_token = self._az_auth_context.get_token(self._scopes[0])
return {TokenConstants.AZ_TOKEN_TYPE: TokenConstants.BEARER_TYPE, TokenConstants.AZ_ACCESS_TOKEN: self._az_token.token}
except Exception as e:
raise KustoClientError(
"Failed to obtain Az Cli token for '{0}'.\nPlease be sure AzCli version 2.3.0 and above is intalled.\n{1}".format(self._kusto_uri, e)
)
async def _get_token_impl_async(self) -> Optional[dict]:
try:
if self._az_auth_context_async is None:
self._az_auth_context_async = AsyncAzureCliCredential()
self._az_token = await self._az_auth_context_async.get_token(self._scopes[0])
return {TokenConstants.AZ_TOKEN_TYPE: TokenConstants.BEARER_TYPE, TokenConstants.AZ_ACCESS_TOKEN: self._az_token.token}
except Exception as e:
raise KustoClientError(
"Failed to obtain Az Cli token for '{0}'.\nPlease be sure AzCli version 2.3.0 and above is installed.\n{1}".format(self._kusto_uri, e)
)
def _get_token_from_cache_impl(self) -> Optional[dict]:
if self._az_token is not None:
# A token is considered valid if it is due to expire in no less than 10 minutes
cur_time = time.time()
if (self._az_token.expires_on - 600) > cur_time:
return {TokenConstants.MSAL_TOKEN_TYPE: TokenConstants.BEARER_TYPE, TokenConstants.MSAL_ACCESS_TOKEN: self._az_token.token}
return None
class UserPassTokenProvider(CloudInfoTokenProvider):
"""Acquire a token from MSAL with username and password"""
def __init__(self, kusto_uri: str, authority_id: str, username: str, password: str, is_async: bool = False):
super().__init__(kusto_uri, is_async)
self._msal_client = None
self._auth = authority_id
self._user = username
self._pass = password
@staticmethod
def name() -> str:
return "UserPassTokenProvider"
def _context_impl(self) -> dict:
return {"authority": self._cloud_info.authority_uri(self._auth), "client_id": self._cloud_info.kusto_client_app_id, "username": self._user}
def _init_impl(self):
self._msal_client = PublicClientApplication(
client_id=self._cloud_info.kusto_client_app_id, authority=self._cloud_info.authority_uri(self._auth), proxies=self._proxy_dict
)
def _get_token_impl(self) -> Optional[dict]:
token = self._msal_client.acquire_token_by_username_password(username=self._user, password=self._pass, scopes=self._scopes)
return self._valid_token_or_throw(token)
def _get_token_from_cache_impl(self) -> dict:
account = None
if self._user is not None:
accounts = self._msal_client.get_accounts(self._user)
if len(accounts) > 0:
account = accounts[0]
token = self._msal_client.acquire_token_silent(scopes=self._scopes, account=account)
return self._valid_token_or_none(token)
class DeviceLoginTokenProvider(CloudInfoTokenProvider):
"""Acquire a token from MSAL with Device Login flow"""
def __init__(self, kusto_uri: str, authority_id: str, device_code_callback=None, is_async: bool = False):
super().__init__(kusto_uri, is_async)
self._msal_client = None
self._auth = authority_id
self._account = None
self._device_code_callback = device_code_callback
@staticmethod
def name() -> str:
return "DeviceLoginTokenProvider"
def _context_impl(self) -> dict:
return {"authority": self._cloud_info.authority_uri(self._auth), "client_id": self._cloud_info.kusto_client_app_id}
def _init_impl(self):
self._msal_client = PublicClientApplication(
client_id=self._cloud_info.kusto_client_app_id, authority=self._cloud_info.authority_uri(self._auth), proxies=self._proxy_dict
)
def _get_token_impl(self) -> Optional[dict]:
flow = self._msal_client.initiate_device_flow(scopes=self._scopes)
try:
if self._device_code_callback:
self._device_code_callback(flow[TokenConstants.MSAL_DEVICE_MSG])
else:
print(flow[TokenConstants.MSAL_DEVICE_MSG])
webbrowser.open(flow[TokenConstants.MSAL_DEVICE_URI])
except KeyError:
raise KustoClientError("Failed to initiate device code flow")
token = self._msal_client.acquire_token_by_device_flow(flow)
# Keep the account for silent login
if self._valid_token_or_none(token) is not None:
accounts = self._msal_client.get_accounts()
if len(accounts) == 1:
self._account = accounts[0]
return self._valid_token_or_throw(token)
def _get_token_from_cache_impl(self) -> dict:
token = self._msal_client.acquire_token_silent(scopes=self._scopes, account=self._account)
return self._valid_token_or_none(token)
class InteractiveLoginTokenProvider(CloudInfoTokenProvider):
"""Acquire a token from MSAL with Device Login flow"""
def __init__(
self,
kusto_uri: str,
authority_id: str,
login_hint: Optional[str] = None,
domain_hint: Optional[str] = None,
is_async: bool = False,
):
super().__init__(kusto_uri, is_async)
self._msal_client = None
self._auth = authority_id
self._login_hint = login_hint
self._domain_hint = domain_hint
self._account = None
@staticmethod
def name() -> str:
return "InteractiveLoginTokenProvider"
def _context_impl(self) -> dict:
return {"authority": self._cloud_info.authority_uri(self._auth), "client_id": self._cloud_info.kusto_client_app_id}
def _init_impl(self):
self._msal_client = PublicClientApplication(
client_id=self._cloud_info.kusto_client_app_id, authority=self._cloud_info.authority_uri(self._auth), proxies=self._proxy_dict
)
def _get_token_impl(self) -> Optional[dict]:
token = self._msal_client.acquire_token_interactive(
scopes=self._scopes, prompt=TokenConstants.MSAL_INTERACTIVE_PROMPT, login_hint=self._login_hint, domain_hint=self._domain_hint
)
return self._valid_token_or_throw(token)
def _get_token_from_cache_impl(self) -> dict:
account = None
accounts = self._msal_client.get_accounts(self._login_hint)
if len(accounts) > 0:
account = accounts[0]
token = self._msal_client.acquire_token_silent(scopes=self._scopes, account=account)
return self._valid_token_or_none(token)
class ApplicationKeyTokenProvider(CloudInfoTokenProvider):
"""Acquire a token from MSAL with application Id and Key"""
def __init__(self, kusto_uri: str, authority_id: str, app_client_id: str, app_key: str, is_async: bool = False):
super().__init__(kusto_uri, is_async)
self._msal_client = None
self._app_client_id = app_client_id
self._app_key = app_key
self._auth = authority_id
@staticmethod
def name() -> str:
return "ApplicationKeyTokenProvider"
def _context_impl(self) -> dict:
return {"authority": self._cloud_info.authority_uri(self._auth), "client_id": self._app_client_id}
def _init_impl(self):
self._msal_client = ConfidentialClientApplication(
client_id=self._app_client_id, client_credential=self._app_key, authority=self._cloud_info.authority_uri(self._auth), proxies=self._proxy_dict
)
def _get_token_impl(self) -> Optional[dict]:
token = self._msal_client.acquire_token_for_client(scopes=self._scopes)
return self._valid_token_or_throw(token)
def _get_token_from_cache_impl(self) -> dict:
token = self._msal_client.acquire_token_silent(scopes=self._scopes, account=None)
return self._valid_token_or_none(token)
class ApplicationCertificateTokenProvider(CloudInfoTokenProvider):
"""
Acquire a token from MSAL using application certificate
Passing the public certificate is optional and will result in Subject Name & Issuer Authentication
"""
def __init__(
self,
kusto_uri: str,
client_id: str,
authority_id: str,
private_cert: str,
thumbprint: str,
public_cert: str = None,
is_async: bool = False,
):
super().__init__(kusto_uri, is_async)
self._msal_client = None
self._auth = authority_id
self._client_id = client_id
self._cert_credentials = {TokenConstants.MSAL_PRIVATE_CERT: private_cert, TokenConstants.MSAL_THUMBPRINT: thumbprint}
if public_cert is not None:
self._cert_credentials[TokenConstants.MSAL_PUBLIC_CERT] = public_cert
@staticmethod
def name() -> str:
return "ApplicationCertificateTokenProvider"
def _context_impl(self) -> dict:
return {
"authority": self._cloud_info.authority_uri(self._auth),
"client_id": self._client_id,
"thumbprint": self._cert_credentials[TokenConstants.MSAL_THUMBPRINT],
}
def _init_impl(self):
self._msal_client = ConfidentialClientApplication(
client_id=self._client_id, client_credential=self._cert_credentials, authority=self._cloud_info.authority_uri(self._auth), proxies=self._proxy_dict
)
def _get_token_impl(self) -> Optional[dict]:
token = self._msal_client.acquire_token_for_client(scopes=self._scopes)
return self._valid_token_or_throw(token)
def _get_token_from_cache_impl(self) -> dict:
token = self._msal_client.acquire_token_silent(scopes=self._scopes, account=None)
return self._valid_token_or_none(token)
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/_version.py 0000664 0000000 0000000 00000000132 14172227630 0026125 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
VERSION = "3.0.1"
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/aio/ 0000775 0000000 0000000 00000000000 14172227630 0024503 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/aio/__init__.py 0000664 0000000 0000000 00000000040 14172227630 0026606 0 ustar 00root root 0000000 0000000 from .client import KustoClient
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/aio/_models.py 0000664 0000000 0000000 00000001116 14172227630 0026476 0 ustar 00root root 0000000 0000000 from typing import AsyncIterator
from azure.kusto.data._models import KustoResultRow, BaseStreamingKustoResultTable
class KustoStreamingResultTable(BaseStreamingKustoResultTable):
"""Async Iterator over a Kusto result table."""
async def __anext__(self) -> KustoResultRow:
try:
row = await self.raw_rows.__anext__()
except StopAsyncIteration:
self.finished = True
raise
self.row_count += 1
return KustoResultRow(self.columns, row)
def __aiter__(self) -> AsyncIterator[KustoResultRow]:
return self
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/aio/client.py 0000664 0000000 0000000 00000013616 14172227630 0026342 0 ustar 00root root 0000000 0000000 import io
from datetime import timedelta
from typing import Union, Optional
from .response import KustoStreamingResponseDataSet
from .._decorators import documented_by, aio_documented_by
from ..aio.streaming_response import StreamingDataSetEnumerator, JsonTokenReader
from ..client import KustoClient as KustoClientSync, _KustoClientBase, KustoConnectionStringBuilder, ClientRequestProperties, ExecuteRequestParams
from ..data_format import DataFormat
from ..exceptions import KustoAioSyntaxError
from ..response import KustoResponseDataSet
try:
from aiohttp import ClientResponse, ClientSession
except ImportError:
raise KustoAioSyntaxError()
@documented_by(KustoClientSync)
class KustoClient(_KustoClientBase):
@documented_by(KustoClientSync.__init__)
def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str]):
super().__init__(kcsb, True)
self._session = ClientSession()
async def __aenter__(self) -> "KustoClient":
return self
def __aexit__(self, exc_type, exc_val, exc_tb):
return self._session.__aexit__(exc_type, exc_val, exc_tb)
@aio_documented_by(KustoClientSync.execute)
async def execute(self, database: str, query: str, properties: ClientRequestProperties = None) -> KustoResponseDataSet:
query = query.strip()
if query.startswith("."):
return await self.execute_mgmt(database, query, properties)
return await self.execute_query(database, query, properties)
@aio_documented_by(KustoClientSync.execute_query)
async def execute_query(self, database: str, query: str, properties: ClientRequestProperties = None) -> KustoResponseDataSet:
return await self._execute(self._query_endpoint, database, query, None, KustoClient._query_default_timeout, properties)
@aio_documented_by(KustoClientSync.execute_mgmt)
async def execute_mgmt(self, database: str, query: str, properties: ClientRequestProperties = None) -> KustoResponseDataSet:
return await self._execute(self._mgmt_endpoint, database, query, None, KustoClient._mgmt_default_timeout, properties)
@aio_documented_by(KustoClientSync.execute_streaming_ingest)
async def execute_streaming_ingest(
self,
database: str,
table: str,
stream: io.IOBase,
stream_format: Union[DataFormat, str],
properties: ClientRequestProperties = None,
mapping_name: str = None,
):
stream_format = stream_format.kusto_value if isinstance(stream_format, DataFormat) else DataFormat[stream_format.upper()].kusto_value
endpoint = self._streaming_ingest_endpoint + database + "/" + table + "?streamFormat=" + stream_format
if mapping_name is not None:
endpoint = endpoint + "&mappingName=" + mapping_name
await self._execute(endpoint, database, None, stream, self._streaming_ingest_default_timeout, properties)
@aio_documented_by(KustoClientSync._execute_streaming_query_parsed)
async def _execute_streaming_query_parsed(
self, database: str, query: str, timeout: timedelta = _KustoClientBase._query_default_timeout, properties: Optional[ClientRequestProperties] = None
) -> StreamingDataSetEnumerator:
response = await self._execute(self._query_endpoint, database, query, None, timeout, properties, stream_response=True)
return StreamingDataSetEnumerator(JsonTokenReader(response.content))
@aio_documented_by(KustoClientSync.execute_streaming_query)
async def execute_streaming_query(
self, database: str, query: str, timeout: timedelta = _KustoClientBase._query_default_timeout, properties: Optional[ClientRequestProperties] = None
) -> KustoStreamingResponseDataSet:
response = await self._execute_streaming_query_parsed(database, query, timeout, properties)
return KustoStreamingResponseDataSet(response)
@aio_documented_by(KustoClientSync._execute)
async def _execute(
self,
endpoint: str,
database: str,
query: Optional[str],
payload: Optional[io.IOBase],
timeout: timedelta,
properties: ClientRequestProperties = None,
stream_response: bool = False,
) -> Union[KustoResponseDataSet, ClientResponse]:
"""Executes given query against this client"""
request_params = ExecuteRequestParams(database, payload, properties, query, timeout, self._request_headers)
json_payload = request_params.json_payload
request_headers = request_params.request_headers
timeout = request_params.timeout
if self._aad_helper:
request_headers["Authorization"] = await self._aad_helper.acquire_authorization_header_async()
response = await self._session.post(endpoint, headers=request_headers, data=payload, json=json_payload, timeout=timeout.seconds, proxy=self._proxy_url)
if stream_response:
try:
response.raise_for_status()
return response
except Exception as e:
try:
response_text = await response.text()
except Exception:
response_text = None
try:
response_json = await response.json()
except Exception:
response_json = None
raise self._handle_http_error(e, endpoint, payload, response, response.status, response_json, response_text)
async with response:
response_json = None
try:
response_json = await response.json()
response.raise_for_status()
except Exception as e:
try:
response_text = await response.text()
except Exception:
response_text = None
raise self._handle_http_error(e, endpoint, payload, response, response.status, response_json, response_text)
return self._kusto_parse_by_endpoint(endpoint, response_json)
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/aio/response.py 0000664 0000000 0000000 00000006723 14172227630 0026723 0 ustar 00root root 0000000 0000000 from typing import List, AsyncIterator, Union
from azure.kusto.data._models import WellKnownDataSet, KustoResultTable, BaseKustoResultTable
from azure.kusto.data.aio._models import KustoStreamingResultTable
from azure.kusto.data.aio.streaming_response import StreamingDataSetEnumerator
from azure.kusto.data.exceptions import KustoStreamingQueryError
from azure.kusto.data.response import BaseKustoResponseDataSet
from azure.kusto.data.streaming_response import FrameType
class KustoStreamingResponseDataSet(BaseKustoResponseDataSet):
_status_column = "Payload"
_error_column = "Level"
_crid_column = "ClientRequestId"
def __init__(self, streamed_data: StreamingDataSetEnumerator):
self._current_table = None
self._skip_incomplete_tables = False
self.tables = []
self.streamed_data = streamed_data
self.finished = False
def iter_primary_results(self) -> "PrimaryResultsIterator":
return PrimaryResultsIterator(self)
def __aiter__(self) -> AsyncIterator[BaseKustoResultTable]:
return self
async def __anext__(self) -> BaseKustoResultTable:
if self.finished:
raise StopAsyncIteration()
if type(self._current_table) == KustoStreamingResultTable and not self._current_table.finished and not self._skip_incomplete_tables:
raise KustoStreamingQueryError(
"Tried retrieving a new primary_result table before the old one was finished. To override call `set_skip_incomplete_tables(True)`"
)
while True:
try:
table = await self.streamed_data.__anext__()
except StopAsyncIteration:
self.finished = True
return
if table["FrameType"] == FrameType.DataTable:
break
if table["TableKind"] == WellKnownDataSet.PrimaryResult.value:
self._current_table = KustoStreamingResultTable(table)
else:
self._current_table = KustoResultTable(table)
self.tables.append(self._current_table)
return self._current_table
def set_skip_incomplete_tables(self, value: bool):
self._skip_incomplete_tables = value
@property
def errors_count(self) -> int:
if not self.finished:
raise KustoStreamingQueryError("Unable to get errors count before reading all of the tables.")
return super().errors_count
def get_exceptions(self) -> List[str]:
if not self.finished:
raise KustoStreamingQueryError("Unable to get errors count before reading all of the tables.")
return super().get_exceptions()
def __getitem__(self, key: Union[int, str]) -> KustoResultTable:
if isinstance(key, int):
return self.tables[key]
try:
return next(t for t in self.tables if t.table_name == key)
except StopIteration:
raise LookupError(key)
def __len__(self) -> int:
return len(self.tables)
class PrimaryResultsIterator:
# This class exists because you can't raise exception from an generator and keep working
def __init__(self, dataset: KustoStreamingResponseDataSet):
self.dataset = dataset
def __aiter__(self) -> AsyncIterator[KustoStreamingResultTable]:
return self
async def __anext__(self) -> KustoStreamingResultTable:
while True:
table = await self.dataset.__anext__()
if type(table) is KustoStreamingResultTable:
return table
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/aio/streaming_response.py 0000664 0000000 0000000 00000023612 14172227630 0030770 0 ustar 00root root 0000000 0000000 from typing import Any, Tuple, Dict, Iterator
import aiohttp
import ijson
from ijson import IncompleteJSONError
from azure.kusto.data._models import WellKnownDataSet
from azure.kusto.data.exceptions import KustoTokenParsingError, KustoUnsupportedApiError, KustoApiError, KustoMultiApiError
from azure.kusto.data.streaming_response import JsonTokenType, FrameType, JsonToken
class JsonTokenReader:
def __init__(self, stream: aiohttp.StreamReader):
self.json_iter = ijson.parse_async(stream, use_float=True)
def __aiter__(self) -> "JsonTokenReader":
return self
def __anext__(self) -> JsonToken:
return self.read_next_token_or_throw()
async def read_next_token_or_throw(self) -> JsonToken:
try:
next_item = await self.json_iter.__anext__()
except IncompleteJSONError:
next_item = None
if next_item is None:
raise KustoTokenParsingError("Unexpected end of stream")
(token_path, token_type, token_value) = next_item
return JsonToken(token_path, JsonTokenType[token_type.upper()], token_value)
async def read_token_of_type(self, *token_types: JsonTokenType) -> JsonToken:
token = await self.read_next_token_or_throw()
if token.token_type not in token_types:
raise KustoTokenParsingError(f"Expected one the following types: '{','.join(t.name for t in token_types)}' , got type {token.token_type}")
return token
async def read_start_object(self) -> JsonToken:
return await self.read_token_of_type(JsonTokenType.START_MAP)
async def read_start_array(self) -> JsonToken:
return await self.read_token_of_type(JsonTokenType.START_ARRAY)
async def read_string(self) -> str:
return (await self.read_token_of_type(JsonTokenType.STRING)).token_value
async def read_boolean(self) -> bool:
return (await self.read_token_of_type(JsonTokenType.BOOLEAN)).token_value
async def read_number(self) -> float:
return (await self.read_token_of_type(JsonTokenType.NUMBER)).token_value
async def skip_children(self, prev_token: JsonToken):
if prev_token.token_type == JsonTokenType.MAP_KEY:
prev_token = await self.read_next_token_or_throw()
if prev_token.token_type in JsonTokenType.start_tokens():
async for potential_end_token in self:
if potential_end_token.token_path == prev_token.token_path and potential_end_token.token_type in JsonTokenType.end_tokens():
break
async def skip_until_property_name(self, name: str) -> JsonToken:
while True:
token = await self.read_token_of_type(JsonTokenType.MAP_KEY)
if token.token_value == name:
return token
await self.skip_children(token)
async def skip_until_any_property_name(self, *names: str) -> JsonToken:
while True:
token = await self.read_token_of_type(JsonTokenType.MAP_KEY)
if token.token_value in names:
return token
await self.skip_children(token)
async def skip_until_property_name_or_end_object(self, *names: str) -> JsonToken:
async for token in self:
if token.token_type == JsonTokenType.END_MAP:
return token
if token.token_type == JsonTokenType.MAP_KEY:
if token.token_value in names:
return token
await self.skip_children(token)
continue
raise Exception(f"Unexpected token {token}")
async def skip_until_token_with_paths(self, *tokens: (JsonTokenType, str)) -> JsonToken:
async for token in self:
if any((token.token_type == t_type and token.token_path == t_path) for (t_type, t_path) in tokens):
return token
await self.skip_children(token)
class StreamingDataSetEnumerator:
def __init__(self, reader: JsonTokenReader):
self.reader = reader
self.done = False
self.started = False
self.started_primary_results = False
self.finished_primary_results = False
def __aiter__(self) -> "StreamingDataSetEnumerator":
return self
async def __anext__(self) -> Dict[str, Any]:
if self.done:
raise StopIteration()
if not self.started:
await self.reader.read_start_array()
self.started = True
token = await self.reader.skip_until_token_with_paths((JsonTokenType.START_MAP, "item"), (JsonTokenType.END_ARRAY, ""))
if token == JsonTokenType.END_ARRAY:
self.done = True
raise StopIteration()
frame_type = await self.read_frame_type()
parsed_frame = await self.parse_frame(frame_type)
is_primary_result = parsed_frame["FrameType"] == FrameType.DataTable and parsed_frame["TableKind"] == WellKnownDataSet.PrimaryResult.value
if is_primary_result:
self.started_primary_results = True
elif self.started_primary_results:
self.finished_primary_results = True
return parsed_frame
async def parse_frame(self, frame_type: FrameType) -> Dict[str, Any]:
if frame_type == FrameType.DataSetHeader:
frame = await self.extract_props(frame_type, ("IsProgressive", JsonTokenType.BOOLEAN), ("Version", JsonTokenType.STRING))
if frame["IsProgressive"]:
raise KustoUnsupportedApiError.progressive_api_unsupported()
return frame
if frame_type in [FrameType.TableHeader, FrameType.TableFragment, FrameType.TableCompletion, FrameType.TableProgress]:
raise KustoUnsupportedApiError.progressive_api_unsupported()
if frame_type == FrameType.DataTable:
props = await self.extract_props(
frame_type,
("TableId", JsonTokenType.NUMBER),
("TableKind", JsonTokenType.STRING),
("TableName", JsonTokenType.STRING),
("Columns", JsonTokenType.START_ARRAY),
)
await self.reader.skip_until_property_name("Rows")
props["Rows"] = self.row_iterator()
if props["TableKind"] != WellKnownDataSet.PrimaryResult.value:
props["Rows"] = [r async for r in props["Rows"]]
return props
if frame_type == FrameType.DataSetCompletion:
res = await self.extract_props(frame_type, ("HasErrors", JsonTokenType.BOOLEAN), ("Cancelled", JsonTokenType.BOOLEAN))
token = await self.reader.skip_until_property_name_or_end_object("OneApiErrors")
if token.token_type != JsonTokenType.END_MAP:
res["OneApiErrors"] = self.parse_array(skip_start=False)
return res
async def row_iterator(self) -> Iterator[list]:
await self.reader.read_token_of_type(JsonTokenType.START_ARRAY)
while True:
token = await self.reader.read_token_of_type(JsonTokenType.START_ARRAY, JsonTokenType.END_ARRAY, JsonTokenType.START_MAP)
if token.token_type == JsonTokenType.START_MAP:
raise KustoMultiApiError([await self.parse_object(skip_start=True)])
if token.token_type == JsonTokenType.END_ARRAY:
return
yield await self.parse_array(skip_start=True)
async def parse_array(self, skip_start: bool) -> list:
if not skip_start:
await self.reader.read_start_array()
arr = []
while True:
token = await self.reader.read_token_of_type(
JsonTokenType.NULL,
JsonTokenType.BOOLEAN,
JsonTokenType.NUMBER,
JsonTokenType.STRING,
JsonTokenType.START_MAP,
JsonTokenType.START_ARRAY,
JsonTokenType.END_ARRAY,
)
if token.token_type == JsonTokenType.END_ARRAY:
return arr
if token.token_type == JsonTokenType.START_MAP:
arr.append(await self.parse_object(skip_start=True))
elif token.token_type == JsonTokenType.START_ARRAY:
arr.append(await self.parse_array(skip_start=True))
else:
arr.append(token.token_value)
async def parse_object(self, skip_start: bool) -> Dict[str, Any]:
if not skip_start:
await self.reader.read_start_object()
obj = {}
while True:
token_prop_name = await self.reader.read_token_of_type(JsonTokenType.MAP_KEY, JsonTokenType.END_MAP)
if token_prop_name.token_type == JsonTokenType.END_MAP:
return obj
prop_name = token_prop_name.token_value
token = await self.reader.read_token_of_type(
JsonTokenType.NULL, JsonTokenType.BOOLEAN, JsonTokenType.NUMBER, JsonTokenType.STRING, JsonTokenType.START_MAP, JsonTokenType.START_ARRAY
)
if token.token_type == JsonTokenType.START_MAP:
obj[prop_name] = await self.parse_object(skip_start=True)
elif token.token_type == JsonTokenType.START_ARRAY:
obj[prop_name] = await self.parse_array(skip_start=True)
else:
obj[prop_name] = token.token_value
async def extract_props(self, frame_type: FrameType, *props: Tuple[str, JsonTokenType]) -> Dict[str, Any]:
result = {"FrameType": frame_type}
props_dict = dict(props)
while props_dict:
name = (await self.reader.skip_until_any_property_name(*props_dict.keys())).token_value
if props_dict[name] == JsonTokenType.START_ARRAY:
result[name] = await self.parse_array(skip_start=False)
else:
result[name] = (await self.reader.read_token_of_type(props_dict[name])).token_value
props_dict.pop(name)
return result
async def read_frame_type(self) -> FrameType:
await self.reader.skip_until_property_name("FrameType")
return FrameType[await self.reader.read_string()]
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/client.py 0000664 0000000 0000000 00000125441 14172227630 0025572 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import abc
import io
import json
import socket
import sys
import uuid
from copy import copy
from datetime import timedelta
from enum import Enum, unique
from typing import TYPE_CHECKING, Union, Callable, Optional, Any, Coroutine, List, Tuple, AnyStr, IO, NoReturn
import requests
from requests import Response
from urllib3.connection import HTTPConnection
from ._version import VERSION
from .data_format import DataFormat
from .exceptions import KustoServiceError, KustoApiError
from .response import KustoResponseDataSetV1, KustoResponseDataSetV2, KustoStreamingResponseDataSet, KustoResponseDataSet
from .security import _AadHelper
from .streaming_response import StreamingDataSetEnumerator, JsonTokenReader
if TYPE_CHECKING:
import aiohttp
import aiohttp.web_exceptions
class KustoConnectionStringBuilder:
"""
Parses Kusto connection strings.
For usages, check out the sample at:
https://github.com/Azure/azure-kusto-python/blob/master/azure-kusto-data/tests/sample.py
"""
@unique
class ValidKeywords(Enum):
"""
Set of properties that can be use in a connection string provided to KustoConnectionStringBuilder.
For a complete list of properties go to https://docs.microsoft.com/en-us/azure/kusto/api/connection-strings/kusto
"""
data_source = "Data Source"
aad_federated_security = "AAD Federated Security"
aad_user_id = "AAD User ID"
password = "Password"
application_client_id = "Application Client Id"
application_key = "Application Key"
application_certificate = "Application Certificate"
application_certificate_thumbprint = "Application Certificate Thumbprint"
public_application_certificate = "Public Application Certificate"
authority_id = "Authority Id"
application_token = "Application Token"
user_token = "User Token"
msi_auth = "MSI Authentication"
msi_params = "MSI Params"
az_cli = "AZ CLI"
interactive_login = "Interactive Login"
login_hint = "Login Hint"
domain_hint = "Domain Hint"
@classmethod
def parse(cls, key: str) -> "KustoConnectionStringBuilder.ValidKeywords":
"""Create a valid keyword."""
key = key.lower().strip()
if key in ["data source", "addr", "address", "network address", "server"]:
return cls.data_source
if key in ["aad user id"]:
return cls.aad_user_id
if key in ["password", "pwd"]:
return cls.password
if key in ["application client id", "appclientid"]:
return cls.application_client_id
if key in ["application key", "appkey"]:
return cls.application_key
if key in ["application certificate"]:
return cls.application_certificate
if key in ["application certificate thumbprint"]:
return cls.application_certificate_thumbprint
if key in ["public application certificate"]:
return cls.public_application_certificate
if key in ["authority id", "authorityid", "authority", "tenantid", "tenant", "tid"]:
return cls.authority_id
if key in ["aad federated security", "federated security", "federated", "fed", "aadfed"]:
return cls.aad_federated_security
if key in ["application token", "apptoken"]:
return cls.application_token
if key in ["user token", "usertoken", "usrtoken"]:
return cls.user_token
if key in ["msi_auth"]:
return cls.msi_auth
if key in ["msi_type"]:
return cls.msi_params
if key in ["az cli"]:
return cls.az_cli
if key in ["interactive login"]:
return cls.interactive_login
if key in ["login hint"]:
return cls.login_hint
if key in ["domain hint"]:
return cls.domain_hint
raise KeyError(key)
def is_secret(self) -> bool:
"""States for each property if it contains secret"""
return self in [self.password, self.application_key, self.application_certificate, self.application_token, self.user_token]
def is_str_type(self) -> bool:
"""States whether a word is of type str or not."""
return self in [
self.aad_user_id,
self.application_certificate,
self.application_certificate_thumbprint,
self.public_application_certificate,
self.application_client_id,
self.data_source,
self.password,
self.application_key,
self.authority_id,
self.application_token,
self.user_token,
self.login_hint,
self.domain_hint,
]
def is_dict_type(self) -> bool:
return self in [self.msi_params]
def is_bool_type(self) -> bool:
"""States whether a word is of type bool or not."""
return self in [self.aad_federated_security, self.msi_auth, self.az_cli, self.interactive_login]
def __init__(self, connection_string: str):
"""
Creates new KustoConnectionStringBuilder.
:param str connection_string: Kusto connection string should be of the format:
https://.kusto.windows.net;AAD User ID="user@microsoft.com";Password=P@ssWord
For more information please look at:
https://kusto.azurewebsites.net/docs/concepts/kusto_connection_strings.html
"""
_assert_value_is_valid(connection_string)
self._internal_dict = {}
self._token_provider = None
self._async_token_provider = None
if connection_string is not None and "=" not in connection_string.partition(";")[0]:
connection_string = "Data Source=" + connection_string
self[self.ValidKeywords.authority_id] = "common"
for kvp_string in connection_string.split(";"):
key, _, value = kvp_string.partition("=")
keyword = self.ValidKeywords.parse(key)
if keyword.is_str_type():
self[keyword] = value.strip()
if keyword.is_bool_type():
if value.strip() in ["True", "true"]:
self[keyword] = True
elif value.strip() in ["False", "false"]:
self[keyword] = False
else:
raise KeyError("Expected aad federated security to be bool. Recieved %s" % value)
def __setitem__(self, key: "Union[KustoConnectionStringBuilder.ValidKeywords, str]", value: Union[str, bool, dict]):
try:
keyword = key if isinstance(key, self.ValidKeywords) else self.ValidKeywords.parse(key)
except KeyError:
raise KeyError("%s is not supported as an item in KustoConnectionStringBuilder" % key)
if value is None:
raise TypeError("Value cannot be None.")
if keyword.is_str_type():
self._internal_dict[keyword] = value.strip()
elif keyword.is_bool_type():
if not isinstance(value, bool):
raise TypeError("Expected %s to be bool" % key)
self._internal_dict[keyword] = value
elif keyword.is_dict_type():
if not isinstance(value, dict):
raise TypeError("Expected %s to be dict" % key)
self._internal_dict[keyword] = value
else:
raise KeyError("KustoConnectionStringBuilder supports only bools and strings.")
@classmethod
def with_aad_user_password_authentication(
cls, connection_string: str, user_id: str, password: str, authority_id: str = "common"
) -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will authenticate with AAD user name and
password.
:param str connection_string: Kusto connection string should be of the format: https://.kusto.windows.net
:param str user_id: AAD user ID.
:param str password: Corresponding password of the AAD user.
:param str authority_id: optional param. defaults to "common"
"""
_assert_value_is_valid(user_id)
_assert_value_is_valid(password)
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb[kcsb.ValidKeywords.aad_user_id] = user_id
kcsb[kcsb.ValidKeywords.password] = password
kcsb[kcsb.ValidKeywords.authority_id] = authority_id
return kcsb
@classmethod
def with_aad_user_token_authentication(cls, connection_string: str, user_token: str) -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will authenticate with AAD application and
a certificate credentials.
:param str connection_string: Kusto connection string should be of the format:
https://.kusto.windows.net
:param str user_token: AAD user token.
"""
_assert_value_is_valid(user_token)
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb[kcsb.ValidKeywords.user_token] = user_token
return kcsb
@classmethod
def with_aad_application_key_authentication(
cls, connection_string: str, aad_app_id: str, app_key: str, authority_id: str
) -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will authenticate with AAD application and key.
:param str connection_string: Kusto connection string should be of the format: https://.kusto.windows.net
:param str aad_app_id: AAD application ID.
:param str app_key: Corresponding key of the AAD application.
:param str authority_id: Authority id (aka Tenant id) must be provided
"""
_assert_value_is_valid(aad_app_id)
_assert_value_is_valid(app_key)
_assert_value_is_valid(authority_id)
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb[kcsb.ValidKeywords.application_client_id] = aad_app_id
kcsb[kcsb.ValidKeywords.application_key] = app_key
kcsb[kcsb.ValidKeywords.authority_id] = authority_id
return kcsb
@classmethod
def with_aad_application_certificate_authentication(
cls, connection_string: str, aad_app_id: str, certificate: str, thumbprint: str, authority_id: str
) -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will authenticate with AAD application using
a certificate.
:param str connection_string: Kusto connection string should be of the format:
https://.kusto.windows.net
:param str aad_app_id: AAD application ID.
:param str certificate: A PEM encoded certificate private key.
:param str thumbprint: hex encoded thumbprint of the certificate.
:param str authority_id: Authority id (aka Tenant id) must be provided
"""
_assert_value_is_valid(aad_app_id)
_assert_value_is_valid(certificate)
_assert_value_is_valid(thumbprint)
_assert_value_is_valid(authority_id)
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb[kcsb.ValidKeywords.application_client_id] = aad_app_id
kcsb[kcsb.ValidKeywords.application_certificate] = certificate
kcsb[kcsb.ValidKeywords.application_certificate_thumbprint] = thumbprint
kcsb[kcsb.ValidKeywords.authority_id] = authority_id
return kcsb
@classmethod
def with_aad_application_certificate_sni_authentication(
cls, connection_string: str, aad_app_id: str, private_certificate: str, public_certificate: str, thumbprint: str, authority_id: str
) -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will authenticate with AAD application using
a certificate Subject Name and Issuer.
:param str connection_string: Kusto connection string should be of the format:
https://.kusto.windows.net
:param str aad_app_id: AAD application ID.
:param str private_certificate: A PEM encoded certificate private key.
:param str public_certificate: A public certificate matching the provided PEM certificate private key.
:param str thumbprint: hex encoded thumbprint of the certificate.
:param str authority_id: Authority id (aka Tenant id) must be provided
"""
_assert_value_is_valid(aad_app_id)
_assert_value_is_valid(private_certificate)
_assert_value_is_valid(public_certificate)
_assert_value_is_valid(thumbprint)
_assert_value_is_valid(authority_id)
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb[kcsb.ValidKeywords.application_client_id] = aad_app_id
kcsb[kcsb.ValidKeywords.application_certificate] = private_certificate
kcsb[kcsb.ValidKeywords.public_application_certificate] = public_certificate
kcsb[kcsb.ValidKeywords.application_certificate_thumbprint] = thumbprint
kcsb[kcsb.ValidKeywords.authority_id] = authority_id
return kcsb
@classmethod
def with_aad_application_token_authentication(cls, connection_string: str, application_token: str) -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will authenticate with AAD application and
an application token.
:param str connection_string: Kusto connection string should be of the format:
https://.kusto.windows.net
:param str application_token: AAD application token.
"""
_assert_value_is_valid(application_token)
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb[kcsb.ValidKeywords.application_token] = application_token
return kcsb
@classmethod
def with_aad_device_authentication(cls, connection_string: str, authority_id: str = "common") -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will authenticate with AAD application and
password.
:param str connection_string: Kusto connection string should be of the format: https://.kusto.windows.net
:param str authority_id: optional param. defaults to "common"
"""
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb[kcsb.ValidKeywords.authority_id] = authority_id
return kcsb
@classmethod
def with_az_cli_authentication(cls, connection_string: str) -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will use existing authenticated az cli profile
password.
:param str connection_string: Kusto connection string should be of the format: https://.kusto.windows.net
"""
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.az_cli] = True
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
return kcsb
@classmethod
def with_aad_managed_service_identity_authentication(
cls, connection_string: str, client_id: str = None, object_id: str = None, msi_res_id: str = None, timeout: int = None
) -> "KustoConnectionStringBuilder":
"""
Creates a KustoConnection string builder that will authenticate with AAD application, using
an application token obtained from a Microsoft Service Identity endpoint. An optional user
assigned application ID can be added to the token.
:param str connection_string: Kusto connection string should be of the format: https://.kusto.windows.net
:param client_id: an optional user assigned identity provided as an Azure ID of a client
:param object_id: an optional user assigned identity provided as an Azure ID of an object
:param msi_res_id: an optional user assigned identity provided as an Azure ID of an MSI resource
:param timeout: an optional timeout (seconds) to wait for an MSI Authentication to occur
"""
kcsb = cls(connection_string)
params = {}
exclusive_pcount = 0
if timeout is not None:
params["connection_timeout"] = timeout
if client_id is not None:
params["client_id"] = client_id
exclusive_pcount += 1
if object_id is not None:
# Until we upgrade azure-identity to version 1.4.1, only client_id is excepted as a hint for user managed service identity
raise ValueError("User Managed Service Identity with object_id is temporarily not supported by azure identity 1.3.1. Please use client_id instead.")
# noinspection PyUnreachableCode
params["object_id"] = object_id
exclusive_pcount += 1
if msi_res_id is not None:
# Until we upgrade azure-identity to version 1.4.1, only client_id is excepted as a hint for user managed service identity
raise ValueError(
"User Managed Service Identity with msi_res_id is temporarily not supported by azure identity 1.3.1. Please use client_id instead."
)
# noinspection PyUnreachableCode
params["msi_res_id"] = msi_res_id
exclusive_pcount += 1
if exclusive_pcount > 1:
raise ValueError("the following parameters are mutually exclusive and can not be provided at the same time: client_uid, object_id, msi_res_id")
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb[kcsb.ValidKeywords.msi_auth] = True
kcsb[kcsb.ValidKeywords.msi_params] = params
return kcsb
@classmethod
def with_token_provider(cls, connection_string: str, token_provider: Callable[[], str]) -> "KustoConnectionStringBuilder":
"""
Create a KustoConnectionStringBuilder that uses a callback function to obtain a connection token
:param str connection_string: Kusto connection string should be of the format: https://.kusto.windows.net
:param token_provider: a parameterless function that returns a valid bearer token for the relevant kusto resource as a string
"""
assert callable(token_provider)
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb._token_provider = token_provider
return kcsb
@classmethod
def with_async_token_provider(
cls,
connection_string: str,
async_token_provider: Callable[[], Coroutine[None, None, str]],
) -> "KustoConnectionStringBuilder":
"""
Create a KustoConnectionStringBuilder that uses an async callback function to obtain a connection token
:param str connection_string: Kusto connection string should be of the format: https://.kusto.windows.net
:param async_token_provider: a parameterless function that after awaiting returns a valid bearer token for the relevant kusto resource as a string
"""
assert callable(async_token_provider)
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
kcsb._async_token_provider = async_token_provider
return kcsb
@classmethod
def with_interactive_login(
cls, connection_string: str, login_hint: Optional[str] = None, domain_hint: Optional[str] = None
) -> "KustoConnectionStringBuilder":
kcsb = cls(connection_string)
kcsb[kcsb.ValidKeywords.interactive_login] = True
kcsb[kcsb.ValidKeywords.aad_federated_security] = True
if login_hint is not None:
kcsb[kcsb.ValidKeywords.login_hint] = login_hint
if domain_hint is not None:
kcsb[kcsb.ValidKeywords.domain_hint] = domain_hint
return kcsb
@property
def data_source(self) -> Optional[str]:
"""The URI specifying the Kusto service endpoint.
For example, https://kuskus.kusto.windows.net or net.tcp://localhost
"""
return self._internal_dict.get(self.ValidKeywords.data_source)
@property
def aad_user_id(self) -> Optional[str]:
"""The username to use for AAD Federated AuthN."""
return self._internal_dict.get(self.ValidKeywords.aad_user_id)
@property
def password(self) -> Optional[str]:
"""The password to use for authentication when username/password authentication is used.
Must be accompanied by UserID property
"""
return self._internal_dict.get(self.ValidKeywords.password)
@property
def application_client_id(self) -> Optional[str]:
"""The application client id to use for authentication when federated
authentication is used.
"""
return self._internal_dict.get(self.ValidKeywords.application_client_id)
@property
def application_key(self) -> Optional[str]:
"""The application key to use for authentication when federated authentication is used"""
return self._internal_dict.get(self.ValidKeywords.application_key)
@property
def application_certificate(self) -> Optional[str]:
"""A PEM encoded certificate private key."""
return self._internal_dict.get(self.ValidKeywords.application_certificate)
@application_certificate.setter
def application_certificate(self, value: str):
self[self.ValidKeywords.application_certificate] = value
@property
def application_certificate_thumbprint(self) -> Optional[str]:
"""hex encoded thumbprint of the certificate."""
return self._internal_dict.get(self.ValidKeywords.application_certificate_thumbprint)
@application_certificate_thumbprint.setter
def application_certificate_thumbprint(self, value: str):
self[self.ValidKeywords.application_certificate_thumbprint] = value
@property
def application_public_certificate(self) -> Optional[str]:
"""A public certificate matching the PEM encoded certificate private key."""
return self._internal_dict.get(self.ValidKeywords.public_application_certificate)
@application_public_certificate.setter
def application_public_certificate(self, value: str):
self[self.ValidKeywords.public_application_certificate] = value
@property
def authority_id(self) -> Optional[str]:
"""The ID of the AAD tenant where the application is configured.
(should be supplied only for non-Microsoft tenant)"""
return self._internal_dict.get(self.ValidKeywords.authority_id)
@authority_id.setter
def authority_id(self, value: str):
self[self.ValidKeywords.authority_id] = value
@property
def aad_federated_security(self) -> Optional[bool]:
"""A Boolean value that instructs the client to perform AAD federated authentication."""
return self._internal_dict.get(self.ValidKeywords.aad_federated_security)
@property
def user_token(self) -> Optional[str]:
"""User token."""
return self._internal_dict.get(self.ValidKeywords.user_token)
@property
def application_token(self) -> Optional[str]:
"""Application token."""
return self._internal_dict.get(self.ValidKeywords.application_token)
@property
def msi_authentication(self) -> Optional[bool]:
"""A value stating the MSI identity type to obtain"""
return self._internal_dict.get(self.ValidKeywords.msi_auth)
@property
def msi_parameters(self) -> Optional[dict]:
"""A user assigned MSI ID to be obtained"""
return self._internal_dict.get(self.ValidKeywords.msi_params)
@property
def az_cli(self) -> Optional[bool]:
return self._internal_dict.get(self.ValidKeywords.az_cli)
@property
def token_provider(self) -> Optional[Callable[[], str]]:
return self._token_provider
@property
def async_token_provider(self) -> Optional[Callable[[], Coroutine[None, None, str]]]:
return self._async_token_provider
@property
def interactive_login(self) -> bool:
val = self._internal_dict.get(self.ValidKeywords.interactive_login)
return val is not None and val
@property
def login_hint(self) -> Optional[str]:
return self._internal_dict.get(self.ValidKeywords.login_hint)
@property
def domain_hint(self) -> Optional[str]:
return self._internal_dict.get(self.ValidKeywords.domain_hint)
def __str__(self) -> str:
dict_copy = self._internal_dict.copy()
for key in dict_copy:
if key.is_secret():
dict_copy[key] = "****"
return self._build_connection_string(dict_copy)
def __repr__(self) -> str:
return self._build_connection_string(self._internal_dict)
def _build_connection_string(self, kcsb_as_dict: dict) -> str:
return ";".join(["{0}={1}".format(word.value, kcsb_as_dict[word]) for word in self.ValidKeywords if word in kcsb_as_dict])
def _assert_value_is_valid(value: str):
if not value or not value.strip():
raise ValueError("Should not be empty")
class ClientRequestProperties:
"""This class is a POD used by client making requests to describe specific needs from the service executing the requests.
For more information please look at: https://docs.microsoft.com/en-us/azure/kusto/api/netfx/request-properties
"""
results_defer_partial_query_failures_option_name = "deferpartialqueryfailures"
request_timeout_option_name = "servertimeout"
no_request_timeout_option_name = "norequesttimeout"
def __init__(self):
self._options = {}
self._parameters = {}
self.client_request_id = None
self.application = None
self.user = None
def set_parameter(self, name: str, value: str):
"""Sets a parameter's value"""
_assert_value_is_valid(name)
self._parameters[name] = value
def has_parameter(self, name: str) -> bool:
"""Checks if a parameter is specified."""
return name in self._parameters
def get_parameter(self, name: str, default_value: str) -> str:
"""Gets a parameter's value."""
return self._parameters.get(name, default_value)
def set_option(self, name: str, value: Any):
"""Sets an option's value"""
_assert_value_is_valid(name)
self._options[name] = value
def has_option(self, name: str) -> bool:
"""Checks if an option is specified."""
return name in self._options
def get_option(self, name: str, default_value: Any) -> str:
"""Gets an option's value."""
return self._options.get(name, default_value)
def to_json(self) -> str:
"""Safe serialization to a JSON string."""
return json.dumps({"Options": self._options, "Parameters": self._parameters}, default=str)
class ExecuteRequestParams:
def __init__(self, database: str, payload: Optional[io.IOBase], properties: ClientRequestProperties, query: str, timeout: timedelta, request_headers: dict):
request_headers = copy(request_headers)
json_payload = None
if not payload:
json_payload = {"db": database, "csl": query}
if properties:
json_payload["properties"] = properties.to_json()
client_request_id_prefix = "KPC.execute;"
request_headers["Content-Type"] = "application/json; charset=utf-8"
else:
if properties:
request_headers.update(json.loads(properties.to_json())["Options"])
# Before 3.0 it was KPC.execute_streaming_ingest, but was changed to align with the other SDKs
client_request_id_prefix = "KPC.executeStreamingIngest;"
request_headers["Content-Encoding"] = "gzip"
request_headers["x-ms-client-request-id"] = client_request_id_prefix + str(uuid.uuid4())
if properties is not None:
if properties.client_request_id is not None:
request_headers["x-ms-client-request-id"] = properties.client_request_id
if properties.application is not None:
request_headers["x-ms-app"] = properties.application
if properties.user is not None:
request_headers["x-ms-user"] = properties.user
if properties.get_option(ClientRequestProperties.no_request_timeout_option_name, False):
timeout = KustoClient._mgmt_default_timeout
else:
timeout = properties.get_option(ClientRequestProperties.request_timeout_option_name, timeout)
timeout = (timeout or KustoClient._mgmt_default_timeout) + KustoClient._client_server_delta
self.json_payload = json_payload
self.request_headers = request_headers
self.timeout = timeout
class HTTPAdapterWithSocketOptions(requests.adapters.HTTPAdapter):
def __init__(self, *args, **kwargs):
self.socket_options = kwargs.pop("socket_options", None)
self.pool_maxsize = kwargs.pop("pool_maxsize", None)
self.max_retries = kwargs.pop("max_retries", None)
super(HTTPAdapterWithSocketOptions, self).__init__(*args, **kwargs)
def init_poolmanager(self, *args, **kwargs):
if self.socket_options is not None:
kwargs["socket_options"] = self.socket_options
super(HTTPAdapterWithSocketOptions, self).init_poolmanager(*args, **kwargs)
class _KustoClientBase(abc.ABC):
API_VERSION = "2019-02-13"
_mgmt_default_timeout = timedelta(hours=1, seconds=30)
_query_default_timeout = timedelta(minutes=4, seconds=30)
_streaming_ingest_default_timeout = timedelta(minutes=10)
_aad_helper: _AadHelper
def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str], is_async):
self._kcsb = kcsb
self._proxy_url: Optional[str] = None
if not isinstance(kcsb, KustoConnectionStringBuilder):
self._kcsb = KustoConnectionStringBuilder(kcsb)
self._kusto_cluster = self._kcsb.data_source
# notice that in this context, federated actually just stands for aad auth, not aad federated auth (legacy code)
self._aad_helper = _AadHelper(self._kcsb, is_async) if self._kcsb.aad_federated_security else None
# Create a session object for connection pooling
self._mgmt_endpoint = "{0}/v1/rest/mgmt".format(self._kusto_cluster)
self._query_endpoint = "{0}/v2/rest/query".format(self._kusto_cluster)
self._streaming_ingest_endpoint = "{0}/v1/rest/ingest/".format(self._kusto_cluster)
self._request_headers = {
"Accept": "application/json",
"Accept-Encoding": "gzip,deflate",
"x-ms-client-version": "Kusto.Python.Client:" + VERSION,
"x-ms-version": self.API_VERSION,
}
def set_proxy(self, proxy_url: str):
self._proxy_url = proxy_url
if self._aad_helper:
self._aad_helper.token_provider.set_proxy(proxy_url)
@staticmethod
def _kusto_parse_by_endpoint(endpoint: str, response_json: Any) -> KustoResponseDataSet:
if endpoint.endswith("v2/rest/query"):
return KustoResponseDataSetV2(response_json)
return KustoResponseDataSetV1(response_json)
@staticmethod
def _handle_http_error(
exception: Exception,
endpoint: Optional[str],
payload: Optional[io.IOBase],
response: "Union[Response, aiohttp.ClientResponse]",
status: int,
response_json: Any,
response_text: Optional[str],
) -> NoReturn:
if status == 404:
if payload:
raise KustoServiceError("The ingestion endpoint does not exist. Please enable streaming ingestion on your cluster.", response) from exception
raise KustoServiceError(f"The requested endpoint '{endpoint}' does not exist.", response) from exception
if payload:
message = f"An error occurred while trying to ingest: Status: {status}, Reason: {response.reason}, Text: {response_text}."
if response_json:
raise KustoApiError(response_json, message, response) from exception
raise KustoServiceError(message, response) from exception
if response_json:
raise KustoApiError(response_json, http_response=response) from exception
if response_text:
raise KustoServiceError(response_text, response) from exception
raise KustoServiceError("Server error response contains no data.", response) from exception
class KustoClient(_KustoClientBase):
"""
Kusto client for Python.
The client is a wrapper around the Kusto REST API.
To read more about it, go to https://docs.microsoft.com/en-us/azure/kusto/api/rest/
The primary methods are:
`execute_query`: executes a KQL query against the Kusto service.
`execute_mgmt`: executes a KQL control command against the Kusto service.
"""
_mgmt_default_timeout = timedelta(hours=1)
_query_default_timeout = timedelta(minutes=4)
_streaming_ingest_default_timeout = timedelta(minutes=10)
_client_server_delta = timedelta(seconds=30)
# The maximum amount of connections to be able to operate in parallel
_max_pool_size = 100
def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str]):
"""
Kusto Client constructor.
:param kcsb: The connection string to initialize KustoClient.
:type kcsb: azure.kusto.data.KustoConnectionStringBuilder or str
"""
super().__init__(kcsb, False)
# Create a session object for connection pooling
self._session = requests.Session()
adapter = HTTPAdapterWithSocketOptions(
socket_options=(HTTPConnection.default_socket_options or []) + self.compose_socket_options(), pool_maxsize=self._max_pool_size
)
self._session.mount("http://", adapter)
self._session.mount("https://", adapter)
def set_proxy(self, proxy_url: str):
super().set_proxy(proxy_url)
self._session.proxies = {"http": proxy_url, "https": proxy_url}
def set_http_retries(self, max_retries: int):
"""
Set the number of HTTP retries to attempt
"""
adapter = HTTPAdapterWithSocketOptions(
socket_options=(HTTPConnection.default_socket_options or []) + self.compose_socket_options(),
pool_maxsize=self._max_pool_size,
max_retries=max_retries,
)
self._session.mount("http://", adapter)
self._session.mount("https://", adapter)
@staticmethod
def compose_socket_options() -> List[Tuple[int, int, int]]:
# Sends TCP Keep-Alive after MAX_IDLE_SECONDS seconds of idleness, once every INTERVAL_SECONDS seconds, and closes the connection after MAX_FAILED_KEEPALIVES failed pings (e.g. 20 => 1:00:30)
MAX_IDLE_SECONDS = 30
INTERVAL_SECONDS = 180 # Corresponds to Azure Load Balancer Service 4 minute timeout, with 1 minute of slack
MAX_FAILED_KEEPALIVES = 20
if (
sys.platform == "linux"
and hasattr(socket, "SOL_SOCKET")
and hasattr(socket, "SO_KEEPALIVE")
and hasattr(socket, "TCP_KEEPIDLE")
and hasattr(socket, "TCP_KEEPINTVL")
and hasattr(socket, "TCP_KEEPCNT")
):
return [
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, MAX_IDLE_SECONDS),
(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, INTERVAL_SECONDS),
(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, MAX_FAILED_KEEPALIVES),
]
elif (
sys.platform == "win32"
and hasattr(socket, "SOL_SOCKET")
and hasattr(socket, "SO_KEEPALIVE")
and hasattr(socket, "TCP_KEEPIDLE")
and hasattr(socket, "TCP_KEEPCNT")
):
return [
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, MAX_IDLE_SECONDS),
(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, MAX_FAILED_KEEPALIVES),
]
elif sys.platform == "darwin" and hasattr(socket, "SOL_SOCKET") and hasattr(socket, "SO_KEEPALIVE") and hasattr(socket, "IPPROTO_TCP"):
TCP_KEEPALIVE = 0x10
return [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), (socket.IPPROTO_TCP, TCP_KEEPALIVE, INTERVAL_SECONDS)]
else:
return []
def execute(self, database: str, query: str, properties: Optional[ClientRequestProperties] = None) -> KustoResponseDataSet:
"""
Executes a query or management command.
:param str database: Database against query will be executed.
:param str query: Query to be executed.
:param azure.kusto.data.ClientRequestProperties properties: Optional additional properties.
:return: Kusto response data set.
:rtype: azure.kusto.data.response.KustoResponseDataSet
"""
query = query.strip()
if query.startswith("."):
return self.execute_mgmt(database, query, properties)
return self.execute_query(database, query, properties)
def execute_query(self, database: str, query: str, properties: Optional[ClientRequestProperties] = None) -> KustoResponseDataSet:
"""
Execute a KQL query.
To learn more about KQL go to https://docs.microsoft.com/en-us/azure/kusto/query/
:param str database: Database against query will be executed.
:param str query: Query to be executed.
:param azure.kusto.data.ClientRequestProperties properties: Optional additional properties.
:return: Kusto response data set.
:rtype: azure.kusto.data.response.KustoResponseDataSet
"""
return self._execute(self._query_endpoint, database, query, None, self._query_default_timeout, properties)
def execute_mgmt(self, database: str, query: str, properties: Optional[ClientRequestProperties] = None) -> KustoResponseDataSet:
"""
Execute a KQL control command.
To learn more about KQL control commands go to https://docs.microsoft.com/en-us/azure/kusto/management/
:param str database: Database against query will be executed.
:param str query: Query to be executed.
:param azure.kusto.data.ClientRequestProperties properties: Optional additional properties.
:return: Kusto response data set.
:rtype: azure.kusto.data.response.KustoResponseDataSet
"""
return self._execute(self._mgmt_endpoint, database, query, None, self._mgmt_default_timeout, properties)
def execute_streaming_ingest(
self,
database: str,
table: str,
stream: IO[AnyStr],
stream_format: Union[DataFormat, str],
properties: Optional[ClientRequestProperties] = None,
mapping_name: str = None,
):
"""
Execute streaming ingest against this client
If the Kusto service is not configured to allow streaming ingestion, this may raise an error
To learn more about streaming ingestion go to:
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-streaming
:param str database: Target database.
:param str table: Target table.
:param io.BaseIO stream: stream object which contains the data to ingest.
:param DataFormat stream_format: Format of the data in the stream.
:param ClientRequestProperties properties: additional request properties.
:param str mapping_name: Pre-defined mapping of the table. Required when stream_format is json/avro.
"""
stream_format = stream_format.kusto_value if isinstance(stream_format, DataFormat) else DataFormat[stream_format.upper()].kusto_value
endpoint = self._streaming_ingest_endpoint + database + "/" + table + "?streamFormat=" + stream_format
if mapping_name is not None:
endpoint = endpoint + "&mappingName=" + mapping_name
self._execute(endpoint, database, None, stream, self._streaming_ingest_default_timeout, properties)
def _execute_streaming_query_parsed(
self, database: str, query: str, timeout: timedelta = _KustoClientBase._query_default_timeout, properties: Optional[ClientRequestProperties] = None
) -> StreamingDataSetEnumerator:
response = self._execute(self._query_endpoint, database, query, None, timeout, properties, stream_response=True)
response.raw.decode_content = True
return StreamingDataSetEnumerator(JsonTokenReader(response.raw))
def execute_streaming_query(
self, database: str, query: str, timeout: timedelta = _KustoClientBase._query_default_timeout, properties: Optional[ClientRequestProperties] = None
) -> KustoStreamingResponseDataSet:
"""
Execute a KQL query without reading it all to memory.
The resulting KustoStreamingResponseDataSet will stream one table at a time, and the rows can be retrieved sequentially.
:param str database: Database against query will be executed.
:param str query: Query to be executed.
:param timedelta timeout: timeout for the query to be executed
:param azure.kusto.data.ClientRequestProperties properties: Optional additional properties.
:return KustoStreamingResponseDataSet:
"""
return KustoStreamingResponseDataSet(self._execute_streaming_query_parsed(database, query, timeout, properties))
def _execute(
self,
endpoint: str,
database: str,
query: Optional[str],
payload: Optional[IO[AnyStr]],
timeout: timedelta,
properties: Optional[ClientRequestProperties] = None,
stream_response: bool = False,
) -> Union[KustoResponseDataSet, Response]:
"""Executes given query against this client"""
request_params = ExecuteRequestParams(database, payload, properties, query, timeout, self._request_headers)
json_payload = request_params.json_payload
request_headers = request_params.request_headers
timeout = request_params.timeout
if self._aad_helper:
request_headers["Authorization"] = self._aad_helper.acquire_authorization_header()
response = self._session.post(endpoint, headers=request_headers, json=json_payload, data=payload, timeout=timeout.seconds, stream=stream_response)
if stream_response:
try:
response.raise_for_status()
return response
except Exception as e:
raise self._handle_http_error(e, self._query_endpoint, None, response, response.status_code, response.json(), response.text)
response_json = None
try:
response_json = response.json()
response.raise_for_status()
except Exception as e:
raise self._handle_http_error(e, endpoint, payload, response, response.status_code, response_json, response.text)
return self._kusto_parse_by_endpoint(endpoint, response_json)
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/data_format.py 0000664 0000000 0000000 00000003626 14172227630 0026575 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from enum import Enum
class IngestionMappingKind(Enum):
CSV = "Csv"
JSON = "Json"
AVRO = "Avro"
APACHEAVRO = "ApacheAvro"
PARQUET = "Parquet"
SSTREAM = "SStream"
ORC = "Orc"
W3CLOGFILE = "W3CLogFile"
UNKNOWN = "Unknown"
class DataFormat(Enum):
"""All data formats supported by Kusto."""
CSV = ("csv", IngestionMappingKind.CSV, False, True)
TSV = ("tsv", IngestionMappingKind.CSV, False, True)
SCSV = ("scsv", IngestionMappingKind.CSV, False, True)
SOHSV = ("sohsv", IngestionMappingKind.CSV, False, True)
PSV = ("psv", IngestionMappingKind.CSV, False, True)
TXT = ("txt", IngestionMappingKind.CSV, False, True)
TSVE = ("tsve", IngestionMappingKind.CSV, False, True)
JSON = ("json", IngestionMappingKind.JSON, True, True)
SINGLEJSON = ("singlejson", IngestionMappingKind.JSON, True, True)
MULTIJSON = ("multijson", IngestionMappingKind.JSON, True, True)
AVRO = ("avro", IngestionMappingKind.AVRO, True, True)
APACHEAVRO = ("apacheavro", IngestionMappingKind.APACHEAVRO, False, True)
PARQUET = ("parquet", IngestionMappingKind.PARQUET, False, False)
SSTREAM = ("sstream", IngestionMappingKind.SSTREAM, False, False)
ORC = ("orc", IngestionMappingKind.ORC, False, False)
RAW = ("raw", IngestionMappingKind.CSV, False, True)
W3CLOGFILE = ("w3clogfile", IngestionMappingKind.W3CLOGFILE, False, True)
def __init__(self, kusto_value: str, ingestion_mapping_kind: IngestionMappingKind, mapping_required: bool, compressible: bool):
self.kusto_value = kusto_value # Formatted how Kusto Service expects it
self.ingestion_mapping_kind = ingestion_mapping_kind
# Deprecated - will probably be removed soon
self._mapping_required = mapping_required
self.compressible = compressible # Binary formats should not be compressed
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/exceptions.py 0000664 0000000 0000000 00000013357 14172227630 0026477 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from typing import List, Union, TYPE_CHECKING, Optional, Dict, Any
if TYPE_CHECKING:
import requests
try:
from aiohttp import ClientResponse
except ImportError:
# No aio installed, ignore
ClientResponse = None
pass
class KustoError(Exception):
"""Base class for all exceptions raised by the Kusto Python Client Libraries."""
class KustoStreamingQueryError(KustoError):
...
class KustoTokenParsingError(KustoStreamingQueryError):
...
class KustoServiceError(KustoError):
"""Raised when the Kusto service was unable to process a request."""
def __init__(
self,
messages: Union[str, List[dict]],
http_response: "Union[requests.Response, ClientResponse, None]" = None,
kusto_response: Optional[Dict[str, Any]] = None,
):
super().__init__(messages)
self.http_response = http_response
self.kusto_response = kusto_response
def get_raw_http_response(self) -> "Union[requests.Response, ClientResponse, None]":
"""Gets the http response."""
return self.http_response
def is_semantic_error(self) -> bool:
"""Checks if a response is a semantic error."""
try:
return "Semantic error:" in self.http_response.text
except AttributeError:
return False
def has_partial_results(self) -> bool:
"""Checks if a response exists."""
return self.kusto_response is not None
def get_partial_results(self) -> Optional[Dict[str, Any]]:
"""Gets the Kusto response."""
return self.kusto_response
class OneApiError:
def __init__(self, code: str, message: str, type: str, description: str, context: dict, permanent: bool) -> None:
self.code = code
self.message = message
self.type = type
self.description = description
self.context = context
self.permanent = permanent
@staticmethod
def from_dict(obj: dict) -> "OneApiError":
code = obj["code"]
message = obj["message"]
type = obj["@type"]
description = obj["@message"]
context = obj["@context"]
permanent = obj["@permanent"]
return OneApiError(code, message, type, description, context, permanent)
class KustoMultiApiError(KustoServiceError):
"""
Represents a collection of standard API errors from kusto. Use `get_api_errors()` to retrieve more details.
"""
def __init__(self, errors: List[dict]):
self.errors = KustoMultiApiError.parse_errors(errors)
messages = [error.description for error in self.errors]
super().__init__(messages[0] if len(self.errors) == 1 else messages)
def get_api_errors(self) -> List[OneApiError]:
return self.errors
@staticmethod
def parse_errors(errors: List[dict]) -> List[OneApiError]:
parsed_errors = []
for error_block in errors:
one_api_errors = error_block.get("OneApiErrors", None)
if not one_api_errors:
continue
for inner_error in one_api_errors:
error_dict = inner_error.get("error", None)
if error_dict:
parsed_errors.append(OneApiError.from_dict(error_dict))
return parsed_errors
class KustoApiError(KustoServiceError):
"""
Represents a standard API error from kusto. Use `get_api_error()` to retrieve more details.
"""
def __init__(self, error_dict: dict, message: str = None, http_response: "Union[requests.Response, ClientResponse, None]" = None, kusto_response=None):
self.error = OneApiError.from_dict(error_dict["error"])
super().__init__(message or self.error.description, http_response, kusto_response)
def get_api_error(self) -> OneApiError:
return self.error
class KustoClientError(KustoError):
"""Raised when a Kusto client is unable to send or complete a request."""
class KustoBlobError(KustoClientError):
def __init__(self, inner: Exception):
self.inner = inner
def message(self) -> str:
return f"Failed to upload blob: {self.inner}"
class KustoUnsupportedApiError(KustoError):
"""Raised when a Kusto client is unable to send or complete a request."""
@staticmethod
def progressive_api_unsupported() -> "KustoUnsupportedApiError":
return KustoUnsupportedApiError("Progressive API is unsupported - to resolve, set results_progressive_enabled=false")
class KustoAuthenticationError(KustoClientError):
"""Raised when authentication fails."""
def __init__(self, authentication_method: str, exception: Exception, **kwargs):
super().__init__()
self.authentication_method = authentication_method
self.exception = exception
if "authority" in kwargs:
self.authority = kwargs["authority"]
if "kusto_uri" in kwargs:
self.kusto_cluster = kwargs["kusto_uri"]
self.kwargs = kwargs
def __str__(self):
return repr(self)
def __repr__(self):
return "KustoAuthenticationError('{}', '{}', '{}')".format(self.authentication_method, repr(self.exception), self.kwargs)
class KustoAioSyntaxError(SyntaxError):
"""Raised when trying to use aio syntax without installing the needed modules"""
def __init__(self):
super().__init__("Aio modules not installed, run 'pip install azure-kusto-data[aio]' to leverage aio capabilities")
class KustoAsyncUsageError(Exception):
"""Raised when trying to use async methods on a sync object, and vice-versa"""
def __init__(self, method: str, is_client_async: bool):
super().__init__("Method {} can't be called from {} client".format(method, "an asynchronous" if is_client_async else "a synchronous"))
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/helpers.py 0000664 0000000 0000000 00000005443 14172227630 0025755 0 ustar 00root root 0000000 0000000 from typing import TYPE_CHECKING, Union
import numpy as np
if TYPE_CHECKING:
import pandas
from azure.kusto.data._models import KustoResultTable, KustoStreamingResultTable
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
def to_pandas_timedelta(raw_value: Union[int, float, str]) -> "pandas.Timedelta":
"""
Transform a raw python value to a pandas timedelta.
"""
import pandas as pd
if isinstance(raw_value, (int, float)):
# https://docs.microsoft.com/en-us/dotnet/api/system.datetime.ticks
# Kusto saves up to ticks, 1 tick == 100 nanoseconds
return pd.to_timedelta(raw_value * 100, unit="ns")
if isinstance(raw_value, str):
# The timespan format Kusto returns is 'd.hh:mm:ss.ssssss' or 'hh:mm:ss.ssssss' or 'hh:mm:ss'
# Pandas expects 'd days hh:mm:ss.ssssss' or 'hh:mm:ss.ssssss' or 'hh:mm:ss'
parts = raw_value.split(":")
if "." not in parts[0]:
return pd.to_timedelta(raw_value)
else:
formatted_value = raw_value.replace(".", " days ", 1)
return pd.to_timedelta(formatted_value)
def dataframe_from_result_table(table: "Union[KustoResultTable, KustoStreamingResultTable]") -> "pandas.DataFrame":
"""Converts Kusto tables into pandas DataFrame.
:param azure.kusto.data._models.KustoResultTable table: Table received from the response.
:return: pandas DataFrame.
"""
import pandas as pd
if not table:
raise ValueError()
from azure.kusto.data._models import KustoResultTable, KustoStreamingResultTable
if not isinstance(table, KustoResultTable) and not isinstance(table, KustoStreamingResultTable):
raise TypeError("Expected KustoResultTable or KustoStreamingResultTable got {}".format(type(table).__name__))
columns = [col.column_name for col in table.columns]
frame = pd.DataFrame(table.raw_rows, columns=columns)
# fix types
for col in table.columns:
if col.column_type == "bool":
frame[col.column_name] = frame[col.column_name].astype(bool)
elif col.column_type == "int" or col.column_type == "long":
frame[col.column_name] = frame[col.column_name].astype("Int64")
elif col.column_type == "real" or col.column_type == "decimal":
frame[col.column_name] = frame[col.column_name].replace("NaN", np.NaN).replace("Infinity", np.PINF).replace("-Infinity", np.NINF)
frame[col.column_name] = pd.to_numeric(frame[col.column_name], errors="coerce").astype("Float64")
elif col.column_type == "datetime":
frame[col.column_name] = pd.to_datetime(frame[col.column_name], errors="coerce")
elif col.column_type == "timespan":
frame[col.column_name] = frame[col.column_name].apply(to_pandas_timedelta)
return frame
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/response.py 0000664 0000000 0000000 00000021255 14172227630 0026150 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from abc import ABCMeta, abstractmethod
from typing import List, Iterator, Union, Dict, Any
from ._models import KustoResultTable, WellKnownDataSet, KustoStreamingResultTable, BaseKustoResultTable
from .exceptions import KustoStreamingQueryError
from .streaming_response import StreamingDataSetEnumerator, FrameType
class BaseKustoResponseDataSet(metaclass=ABCMeta):
tables: list
tables_count: int
tables_names: list
@property
@abstractmethod
def _error_column(self) -> str:
raise NotImplementedError
@property
@abstractmethod
def _crid_column(self) -> str:
raise NotImplementedError
@property
@abstractmethod
def _status_column(self) -> str:
raise NotImplementedError
@property
def errors_count(self) -> int:
"""Checks whether an exception was thrown."""
query_status_table = next((t for t in self.tables if t.table_kind == WellKnownDataSet.QueryCompletionInformation), None)
if not query_status_table:
return 0
min_level = 4
errors = 0
for row in query_status_table:
if row[self._error_column] < 4:
if row[self._error_column] < min_level:
min_level = row[self._error_column]
errors = 1
elif row[self._error_column] == min_level:
errors += 1
return errors
def get_exceptions(self) -> List[str]:
"""Gets the exceptions retrieved from Kusto if exists."""
query_status_table = next((t for t in self.tables if t.table_kind == WellKnownDataSet.QueryCompletionInformation), None)
if not query_status_table:
return []
result = []
for row in query_status_table:
if row[self._error_column] < 4:
result.append(
"Please provide the following data to Kusto: CRID='{0}' Description:'{1}'".format(row[self._crid_column], row[self._status_column])
)
return result
def __iter__(self) -> Iterator[BaseKustoResultTable]:
return iter(self.tables)
def __getitem__(self, key: Union[int, str]) -> KustoResultTable:
if isinstance(key, int):
return self.tables[key]
try:
return self.tables[self.tables_names.index(key)]
except ValueError:
raise LookupError(key)
def __len__(self) -> int:
return self.tables_count
class KustoResponseDataSet(BaseKustoResponseDataSet, metaclass=ABCMeta):
"""
`KustoResponseDataSet` Represents the parsed data set carried by the response to a Kusto request.
`KustoResponseDataSet` provides convenient methods to work with the returned result.
The result table(s) are accessable via the @primary_results property.
@primary_results returns a collection of `KustoResultTable`.
It can contain more than one table when [`fork`](https://docs.microsoft.com/en-us/azure/kusto/query/forkoperator) is used.
"""
def __init__(self, json_response: List[Dict[str, Any]]):
self.tables = [KustoResultTable(t) for t in json_response]
self.tables_count = len(self.tables)
self.tables_names = [t.table_name for t in self.tables]
@property
def primary_results(self) -> List[KustoResultTable]:
"""Returns primary results. If there is more than one returns a list."""
if self.tables_count == 1:
return self.tables
primary = [x for x in self.tables if x.table_kind == WellKnownDataSet.PrimaryResult]
return primary
def __iter__(self) -> Iterator[KustoResultTable]:
return iter(self.tables)
class KustoResponseDataSetV1(KustoResponseDataSet):
"""
KustoResponseDataSetV1 is a wrapper for a V1 Kusto response.
It parses V1 response into a convenient KustoResponseDataSet.
To read more about V1 response structure, please check out https://docs.microsoft.com/en-us/azure/kusto/api/rest/response
"""
_status_column = "StatusDescription"
_crid_column = "ClientActivityId"
_error_column = "Severity"
_tables_kinds = {
"QueryResult": WellKnownDataSet.PrimaryResult,
"QueryProperties": WellKnownDataSet.QueryProperties,
"QueryStatus": WellKnownDataSet.QueryCompletionInformation,
}
def __init__(self, json_response: dict):
super(KustoResponseDataSetV1, self).__init__(json_response["Tables"])
if self.tables_count <= 2:
self.tables[0].table_kind = WellKnownDataSet.PrimaryResult
self.tables[0].table_id = 0
if self.tables_count == 2:
self.tables[1].table_kind = WellKnownDataSet.QueryProperties
self.tables[1].table_id = 1
else:
toc = self.tables[-1]
toc.table_kind = WellKnownDataSet.TableOfContents
toc.table_id = self.tables_count - 1
for i in range(self.tables_count - 1):
self.tables[i].table_name = toc[i]["Name"]
self.tables[i].table_id = toc[i]["Id"]
self.tables[i].table_kind = self._tables_kinds[toc[i]["Kind"]]
class KustoResponseDataSetV2(KustoResponseDataSet):
"""
KustoResponseDataSetV2 is a wrapper for a V2 Kusto response.
It parses V2 response into a convenient KustoResponseDataSet.
To read more about V2 response structure, please check out https://docs.microsoft.com/en-us/azure/kusto/api/rest/response2
"""
_status_column = "Payload"
_error_column = "Level"
_crid_column = "ClientRequestId"
def __init__(self, json_response: List[dict]):
super(KustoResponseDataSetV2, self).__init__([t for t in json_response if t["FrameType"] == "DataTable"])
class KustoStreamingResponseDataSet(BaseKustoResponseDataSet):
_status_column = "Payload"
_error_column = "Level"
_crid_column = "ClientRequestId"
def __init__(self, streamed_data: StreamingDataSetEnumerator):
self._current_table = None
self._skip_incomplete_tables = False
self.tables = []
self.streamed_data = streamed_data
self.finished = False
def iter_primary_results(self) -> "PrimaryResultsIterator":
return PrimaryResultsIterator(self)
def __iter__(self) -> Iterator[Union[KustoResultTable, KustoStreamingResultTable]]:
return self
def __next__(self) -> Union[KustoResultTable, KustoStreamingResultTable]:
if self.finished:
raise StopIteration
if type(self._current_table) is KustoStreamingResultTable and not self._current_table.finished and not self._skip_incomplete_tables:
raise KustoStreamingQueryError(
"Tried retrieving a new primary_result table before the old one was finished. To override call `set_skip_incomplete_tables(True)`"
)
while True:
try:
table = next(self.streamed_data)
except StopIteration:
self.finished = True
raise
if table["FrameType"] == FrameType.DataTable:
break
if table["TableKind"] == WellKnownDataSet.PrimaryResult.value:
self._current_table = KustoStreamingResultTable(table)
else:
self._current_table = KustoResultTable(table)
self.tables.append(self._current_table)
return self._current_table
def set_skip_incomplete_tables(self, value: bool):
self._skip_incomplete_tables = value
@property
def errors_count(self) -> int:
if not self.finished:
raise KustoStreamingQueryError("Unable to get errors count before reading all of the tables.")
return super().errors_count
def get_exceptions(self) -> List[str]:
if not self.finished:
raise KustoStreamingQueryError("Unable to get errors count before reading all of the tables.")
return super().get_exceptions()
def __getitem__(self, key) -> KustoResultTable:
if isinstance(key, int):
return self.tables[key]
try:
return next(t for t in self.tables if t.table_name == key)
except StopIteration:
raise LookupError(key)
def __len__(self) -> int:
return len(self.tables)
class PrimaryResultsIterator:
# This class exists because you can't raise exception from an generator and keep working
def __init__(self, dataset: KustoStreamingResponseDataSet):
self.dataset = dataset
def __iter__(self) -> Iterator[KustoStreamingResultTable]:
return self
def __next__(self) -> KustoStreamingResultTable:
while True:
table = next(self.dataset)
if type(table) is KustoStreamingResultTable:
return table
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/security.py 0000664 0000000 0000000 00000010503 14172227630 0026153 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from typing import Optional, Dict, TYPE_CHECKING
from urllib.parse import urlparse
from ._token_providers import (
TokenProviderBase,
BasicTokenProvider,
CallbackTokenProvider,
MsiTokenProvider,
AzCliTokenProvider,
UserPassTokenProvider,
DeviceLoginTokenProvider,
InteractiveLoginTokenProvider,
ApplicationKeyTokenProvider,
ApplicationCertificateTokenProvider,
TokenConstants,
)
from .exceptions import KustoAuthenticationError, KustoClientError
if TYPE_CHECKING:
from . import KustoConnectionStringBuilder
class _AadHelper:
kusto_uri = None # type: str
authority_uri = None # type: str
token_provider = None # type: TokenProviderBase
def __init__(self, kcsb: "KustoConnectionStringBuilder", is_async: bool):
self.kusto_uri = "{0.scheme}://{0.hostname}".format(urlparse(kcsb.data_source))
self.username = None
if kcsb.interactive_login:
self.token_provider = InteractiveLoginTokenProvider(self.kusto_uri, kcsb.authority_id, kcsb.login_hint, kcsb.domain_hint, is_async=is_async)
elif all([kcsb.aad_user_id, kcsb.password]):
self.token_provider = UserPassTokenProvider(self.kusto_uri, kcsb.authority_id, kcsb.aad_user_id, kcsb.password, is_async=is_async)
elif all([kcsb.application_client_id, kcsb.application_key]):
self.token_provider = ApplicationKeyTokenProvider(
self.kusto_uri, kcsb.authority_id, kcsb.application_client_id, kcsb.application_key, is_async=is_async
)
elif all([kcsb.application_client_id, kcsb.application_certificate, kcsb.application_certificate_thumbprint]):
# kcsb.application_public_certificate can be None if SNI is not used
self.token_provider = ApplicationCertificateTokenProvider(
self.kusto_uri,
kcsb.application_client_id,
kcsb.authority_id,
kcsb.application_certificate,
kcsb.application_certificate_thumbprint,
kcsb.application_public_certificate,
is_async=is_async,
)
elif kcsb.msi_authentication:
self.token_provider = MsiTokenProvider(self.kusto_uri, kcsb.msi_parameters, is_async=is_async)
elif kcsb.user_token:
self.token_provider = BasicTokenProvider(kcsb.user_token, is_async=is_async)
elif kcsb.application_token:
self.token_provider = BasicTokenProvider(kcsb.application_token, is_async=is_async)
elif kcsb.az_cli:
self.token_provider = AzCliTokenProvider(self.kusto_uri, is_async=is_async)
elif kcsb.token_provider or kcsb.async_token_provider:
self.token_provider = CallbackTokenProvider(token_callback=kcsb.token_provider, async_token_callback=kcsb.async_token_provider, is_async=is_async)
else:
self.token_provider = DeviceLoginTokenProvider(self.kusto_uri, kcsb.authority_id, is_async=is_async)
def acquire_authorization_header(self):
try:
return _get_header_from_dict(self.token_provider.get_token())
except Exception as error:
kwargs = self.token_provider.context()
kwargs["kusto_uri"] = self.kusto_uri
raise KustoAuthenticationError(self.token_provider.name(), error, **kwargs)
async def acquire_authorization_header_async(self):
try:
return _get_header_from_dict(await self.token_provider.get_token_async())
except Exception as error:
kwargs = await self.token_provider.context_async()
kwargs["resource"] = self.kusto_uri
raise KustoAuthenticationError(self.token_provider.name(), error, **kwargs)
def _get_header_from_dict(token: dict):
if TokenConstants.MSAL_ACCESS_TOKEN in token:
return _get_header(token[TokenConstants.MSAL_TOKEN_TYPE], token[TokenConstants.MSAL_ACCESS_TOKEN])
elif TokenConstants.AZ_ACCESS_TOKEN in token:
return _get_header(token[TokenConstants.AZ_TOKEN_TYPE], token[TokenConstants.AZ_ACCESS_TOKEN])
else:
raise KustoClientError("Unable to determine the token type. Neither 'tokenType' nor 'token_type' property is present.")
def _get_header(token_type: str, access_token: str) -> str:
return "{0} {1}".format(token_type, access_token)
azure-kusto-python-3.0.1/azure-kusto-data/azure/kusto/data/streaming_response.py 0000664 0000000 0000000 00000024762 14172227630 0030227 0 ustar 00root root 0000000 0000000 from enum import Enum
from typing import Optional, Any, Tuple, Dict, AnyStr, IO, List, Iterator
import ijson
from ijson import IncompleteJSONError
from azure.kusto.data._models import WellKnownDataSet
from azure.kusto.data.exceptions import KustoServiceError, KustoTokenParsingError, KustoUnsupportedApiError, KustoApiError, KustoMultiApiError
class JsonTokenType(Enum):
NULL = 0
BOOLEAN = 1
NUMBER = 2
STRING = 3
MAP_KEY = 4
START_MAP = 5
END_MAP = 6
START_ARRAY = 7
END_ARRAY = 8
@staticmethod
def start_tokens() -> "List[JsonTokenType]":
return [JsonTokenType.START_MAP, JsonTokenType.START_ARRAY]
@staticmethod
def end_tokens() -> "List[JsonTokenType]":
return [JsonTokenType.END_MAP, JsonTokenType.END_ARRAY]
class FrameType(Enum):
DataSetHeader = 0
TableHeader = 1
TableFragment = 2
TableCompletion = 3
TableProgress = 4
DataTable = 5
DataSetCompletion = 6
class JsonToken:
def __init__(self, token_path: str, token_type: JsonTokenType, token_value: Optional[Any]):
self.token_path = token_path
self.token_type = token_type
self.token_value = token_value
class JsonTokenReader:
def __init__(self, stream: IO[AnyStr]):
self.json_iter = ijson.parse(stream, use_float=True)
def __iter__(self) -> "JsonTokenReader":
return self
def __next__(self) -> JsonToken:
return self.read_next_token_or_throw()
def read_next_token_or_throw(self) -> JsonToken:
try:
next_item = next(self.json_iter)
except IncompleteJSONError:
next_item = None
if next_item is None:
raise KustoTokenParsingError("Unexpected end of stream")
(token_path, token_type, token_value) = next_item
return JsonToken(token_path, JsonTokenType[token_type.upper()], token_value)
def read_token_of_type(self, *token_types: JsonTokenType) -> JsonToken:
token = self.read_next_token_or_throw()
if token.token_type not in token_types:
raise KustoTokenParsingError(f"Expected one the following types: '{','.join(t.name for t in token_types)}' , got type {token.token_type}")
return token
def read_start_object(self) -> JsonToken:
return self.read_token_of_type(JsonTokenType.START_MAP)
def read_start_array(self) -> JsonToken:
return self.read_token_of_type(JsonTokenType.START_ARRAY)
def read_string(self) -> str:
return self.read_token_of_type(JsonTokenType.STRING).token_value
def read_boolean(self) -> bool:
return self.read_token_of_type(JsonTokenType.BOOLEAN).token_value
def read_number(self) -> float:
return self.read_token_of_type(JsonTokenType.NUMBER).token_value
def skip_children(self, prev_token: JsonToken):
if prev_token.token_type == JsonTokenType.MAP_KEY:
prev_token = self.read_next_token_or_throw()
if prev_token.token_type in JsonTokenType.start_tokens():
for potential_end_token in self:
if potential_end_token.token_path == prev_token.token_path and potential_end_token.token_type in JsonTokenType.end_tokens():
break
def skip_until_property_name(self, name: str) -> JsonToken:
while True:
token = self.read_token_of_type(JsonTokenType.MAP_KEY)
if token.token_value == name:
return token
self.skip_children(token)
def skip_until_any_property_name(self, *names: str) -> JsonToken:
while True:
token = self.read_token_of_type(JsonTokenType.MAP_KEY)
if token.token_value in names:
return token
self.skip_children(token)
def skip_until_property_name_or_end_object(self, *names: str) -> JsonToken:
for token in self:
if token.token_type == JsonTokenType.END_MAP:
return token
if token.token_type == JsonTokenType.MAP_KEY:
if token.token_value in names:
return token
self.skip_children(token)
continue
raise Exception(f"Unexpected token {token}")
def skip_until_token_with_paths(self, *tokens: (JsonTokenType, str)) -> JsonToken:
for token in self:
if any((token.token_type == t_type and token.token_path == t_path) for (t_type, t_path) in tokens):
return token
self.skip_children(token)
class StreamingDataSetEnumerator:
def __init__(self, reader: JsonTokenReader):
self.reader = reader
self.done = False
self.started = False
self.started_primary_results = False
self.finished_primary_results = False
def __iter__(self) -> "StreamingDataSetEnumerator":
return self
def __next__(self) -> Dict[str, Any]:
if self.done:
raise StopIteration()
if not self.started:
self.reader.read_start_array()
self.started = True
token = self.reader.skip_until_token_with_paths((JsonTokenType.START_MAP, "item"), (JsonTokenType.END_ARRAY, ""))
if token == JsonTokenType.END_ARRAY:
self.done = True
raise StopIteration()
frame_type = self.read_frame_type()
parsed_frame = self.parse_frame(frame_type)
is_primary_result = parsed_frame["FrameType"] == FrameType.DataTable and parsed_frame["TableKind"] == WellKnownDataSet.PrimaryResult.value
if is_primary_result:
self.started_primary_results = True
elif self.started_primary_results:
self.finished_primary_results = True
return parsed_frame
def parse_frame(self, frame_type: FrameType) -> Dict[str, Any]:
if frame_type == FrameType.DataSetHeader:
frame = self.extract_props(frame_type, ("IsProgressive", JsonTokenType.BOOLEAN), ("Version", JsonTokenType.STRING))
if frame["IsProgressive"]:
raise KustoUnsupportedApiError.progressive_api_unsupported()
return frame
if frame_type in [FrameType.TableHeader, FrameType.TableFragment, FrameType.TableCompletion, FrameType.TableProgress]:
raise KustoUnsupportedApiError.progressive_api_unsupported()
if frame_type == FrameType.DataTable:
props = self.extract_props(
frame_type,
("TableId", JsonTokenType.NUMBER),
("TableKind", JsonTokenType.STRING),
("TableName", JsonTokenType.STRING),
("Columns", JsonTokenType.START_ARRAY),
)
self.reader.skip_until_property_name("Rows")
props["Rows"] = self.row_iterator()
if props["TableKind"] != WellKnownDataSet.PrimaryResult.value:
props["Rows"] = list(props["Rows"])
return props
if frame_type == FrameType.DataSetCompletion:
res = self.extract_props(frame_type, ("HasErrors", JsonTokenType.BOOLEAN), ("Cancelled", JsonTokenType.BOOLEAN))
token = self.reader.skip_until_property_name_or_end_object("OneApiErrors")
if token.token_type != JsonTokenType.END_MAP:
res["OneApiErrors"] = self.parse_array(skip_start=False)
return res
def row_iterator(self) -> Iterator[list]:
self.reader.read_token_of_type(JsonTokenType.START_ARRAY)
while True:
token = self.reader.read_token_of_type(JsonTokenType.START_ARRAY, JsonTokenType.END_ARRAY, JsonTokenType.START_MAP)
if token.token_type == JsonTokenType.START_MAP:
# Todo - this method of error handling may be problematic, since after raising an error the iteration stops.
# This means that if there are more data or even more errors, we can't read them
raise KustoMultiApiError([self.parse_object(skip_start=True)])
if token.token_type == JsonTokenType.END_ARRAY:
return
yield self.parse_array(skip_start=True)
def parse_array(self, skip_start: bool) -> list:
if not skip_start:
self.reader.read_start_array()
arr = []
while True:
token = self.reader.read_token_of_type(
JsonTokenType.NULL,
JsonTokenType.BOOLEAN,
JsonTokenType.NUMBER,
JsonTokenType.STRING,
JsonTokenType.START_MAP,
JsonTokenType.START_ARRAY,
JsonTokenType.END_ARRAY,
)
if token.token_type == JsonTokenType.END_ARRAY:
return arr
if token.token_type == JsonTokenType.START_MAP:
arr.append(self.parse_object(skip_start=True))
elif token.token_type == JsonTokenType.START_ARRAY:
arr.append(self.parse_array(skip_start=True))
else:
arr.append(token.token_value)
def parse_object(self, skip_start: bool) -> Dict[str, Any]:
if not skip_start:
self.reader.read_start_object()
obj = {}
while True:
token_prop_name = self.reader.read_token_of_type(JsonTokenType.MAP_KEY, JsonTokenType.END_MAP)
if token_prop_name.token_type == JsonTokenType.END_MAP:
return obj
prop_name = token_prop_name.token_value
token = self.reader.read_token_of_type(
JsonTokenType.NULL, JsonTokenType.BOOLEAN, JsonTokenType.NUMBER, JsonTokenType.STRING, JsonTokenType.START_MAP, JsonTokenType.START_ARRAY
)
if token.token_type == JsonTokenType.START_MAP:
obj[prop_name] = self.parse_object(skip_start=True)
elif token.token_type == JsonTokenType.START_ARRAY:
obj[prop_name] = self.parse_array(skip_start=True)
else:
obj[prop_name] = token.token_value
def extract_props(self, frame_type: FrameType, *props: Tuple[str, JsonTokenType]) -> Dict[str, Any]:
result = {"FrameType": frame_type}
props_dict = dict(props)
while props_dict:
name = self.reader.skip_until_any_property_name(*props_dict.keys()).token_value
if props_dict[name] == JsonTokenType.START_ARRAY:
result[name] = self.parse_array(skip_start=False)
else:
result[name] = self.reader.read_token_of_type(props_dict[name]).token_value
props_dict.pop(name)
return result
def read_frame_type(self) -> FrameType:
self.reader.skip_until_property_name("FrameType")
return FrameType[self.reader.read_string()]
azure-kusto-python-3.0.1/azure-kusto-data/setup.cfg 0000664 0000000 0000000 00000000227 14172227630 0022351 0 ustar 00root root 0000000 0000000 [bdist_wheel]
universal=1
[flake8]
ignore = E226,E302,E41
max-line-length = 160
exclude = tests/*
max-complexity = 10
[pylint]
max-line-length = 160 azure-kusto-python-3.0.1/azure-kusto-data/setup.py 0000664 0000000 0000000 00000003304 14172227630 0022241 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import re
from os import path
# Always prefer setuptools over distutils
from setuptools import setup, find_packages
PACKAGE_NAME = "azure-kusto-data"
# a-b-c => a/b/c
PACKAGE_FOLDER_PATH = PACKAGE_NAME.replace("-", path.sep)
# a-b-c => a.b.c
NAMESPACE_NAME = PACKAGE_NAME.replace("-", ".")
with open(path.join(PACKAGE_FOLDER_PATH, "_version.py"), "r") as fd:
VERSION = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1)
if not VERSION:
raise RuntimeError("Cannot find version information")
setup(
name=PACKAGE_NAME,
version=VERSION,
description="Kusto Data Client",
long_description=open("README.rst", "r").read(),
url="https://github.com/Azure/azure-kusto-python",
author="Microsoft Corporation",
author_email="kustalk@microsoft.com",
license="MIT",
classifiers=[
# 5 - Production/Stable depends on multi-threading / aio / perf
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"Topic :: Software Development",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"License :: OSI Approved :: MIT License",
],
namespace_packages=["azure"],
keywords="kusto wrapper client library",
packages=find_packages(exclude=["azure", "tests"]),
install_requires=["python-dateutil>=2.8.0", "requests>=2.13.0", "azure-identity>=1.5.0,<2", "msal>=1.9.0,<2", "ijson~=3.1"],
extras_require={"pandas": ["pandas"], "aio": ["aiohttp>=3.4.4,<4", "asgiref>=3.2.3,<4"]},
)
azure-kusto-python-3.0.1/azure-kusto-data/tests/ 0000775 0000000 0000000 00000000000 14172227630 0021671 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/tests/__init__.py 0000664 0000000 0000000 00000000000 14172227630 0023770 0 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/tests/aio/ 0000775 0000000 0000000 00000000000 14172227630 0022441 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/tests/aio/__init__.py 0000664 0000000 0000000 00000000000 14172227630 0024540 0 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/tests/aio/test_async_token_providers.py 0000664 0000000 0000000 00000034142 14172227630 0030470 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import os
import pytest
from azure.kusto.data._cloud_settings import CloudInfo
from azure.kusto.data._decorators import aio_documented_by
from azure.kusto.data._token_providers import *
from .test_kusto_client import run_aio_tests
from ..test_token_providers import KUSTO_URI, TOKEN_VALUE, TEST_AZ_AUTH, TEST_MSI_AUTH, TEST_DEVICE_AUTH, TokenProviderTests, MockProvider
@pytest.mark.skipif(not run_aio_tests, reason="requires aio")
@aio_documented_by(TokenProviderTests)
class TestTokenProvider:
@aio_documented_by(TokenProviderTests.test_base_provider)
@pytest.mark.asyncio
async def test_base_provider(self):
# test init with no URI
provider = MockProvider(is_async=True)
# Test provider with URI, No silent token
provider = MockProvider(is_async=True)
token = provider._get_token_from_cache_impl()
assert provider.init_count == 0
assert token is None
token = await provider.get_token_async()
assert provider.init_count == 1
assert TokenConstants.MSAL_ACCESS_TOKEN in token
token = provider._get_token_from_cache_impl()
assert TokenConstants.MSAL_ACCESS_TOKEN in token
token = await provider.get_token_async()
assert provider.init_count == 1
good_token = {TokenConstants.MSAL_ACCESS_TOKEN: TOKEN_VALUE}
bad_token1 = None
bad_token2 = {"error": "something bad occurred"}
assert provider._valid_token_or_none(good_token) == good_token
assert provider._valid_token_or_none(bad_token1) is None
assert provider._valid_token_or_none(bad_token2) is None
assert provider._valid_token_or_throw(good_token) == good_token
exception_occurred = False
try:
provider._valid_token_or_throw(bad_token1)
except KustoClientError:
exception_occurred = True
finally:
assert exception_occurred
exception_occurred = False
try:
provider._valid_token_or_throw(bad_token2)
except KustoClientError:
exception_occurred = True
finally:
assert exception_occurred
@aio_documented_by(TokenProviderTests.get_token_value)
def get_token_value(self, token: dict):
assert token is not None
assert TokenConstants.MSAL_ERROR not in token
value = None
if TokenConstants.MSAL_ACCESS_TOKEN in token:
return token[TokenConstants.MSAL_ACCESS_TOKEN]
elif TokenConstants.AZ_ACCESS_TOKEN in token:
return token[TokenConstants.AZ_ACCESS_TOKEN]
else:
assert False
@staticmethod
def test_fail_async_call():
provider = BasicTokenProvider(token=TOKEN_VALUE, is_async=True)
try:
provider.get_token()
assert False, "Expected KustoAsyncUsageError to occur"
except KustoAsyncUsageError as e:
assert str(e) == "Method get_token can't be called from an asynchronous client"
try:
provider.context()
assert False, "Expected KustoAsyncUsageError to occur"
except KustoAsyncUsageError as e:
assert str(e) == "Method context can't be called from an asynchronous client"
@aio_documented_by(TokenProviderTests.test_basic_provider)
@pytest.mark.asyncio
async def test_basic_provider(self):
provider = BasicTokenProvider(token=TOKEN_VALUE, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) == TOKEN_VALUE
@aio_documented_by(TokenProviderTests.test_callback_token_provider)
@pytest.mark.asyncio
async def test_callback_token_provider(self):
provider = CallbackTokenProvider(token_callback=lambda: TOKEN_VALUE, async_token_callback=None, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) == TOKEN_VALUE
provider = CallbackTokenProvider(token_callback=lambda: 0, async_token_callback=None, is_async=True) # token is not a string
exception_occurred = False
try:
await provider.get_token_async()
except KustoClientError:
exception_occurred = True
finally:
assert exception_occurred
@pytest.mark.asyncio
async def test_callback_token_provider_with_async_method(self):
async def callback():
return TOKEN_VALUE
provider = CallbackTokenProvider(token_callback=None, async_token_callback=callback, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) == TOKEN_VALUE
async def fail_callback():
return 0
provider = CallbackTokenProvider(token_callback=None, async_token_callback=fail_callback, is_async=True) # token is not a string
exception_occurred = False
try:
await provider.get_token_async()
except KustoClientError:
exception_occurred = True
finally:
assert exception_occurred
@aio_documented_by(TokenProviderTests.test_az_provider)
@pytest.mark.asyncio
async def test_az_provider(self):
if not TEST_AZ_AUTH:
print(" *** Skipped Az-Cli Provider Test ***")
return
print("Note!\nThe test 'test_az_provider' will fail if 'az login' was not called.")
provider = AzCliTokenProvider(KUSTO_URI, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
# another run to pass through the cache
token = provider._get_token_from_cache_impl()
assert self.get_token_value(token) is not None
@aio_documented_by(TokenProviderTests.test_msi_provider)
@pytest.mark.asyncio
async def test_msi_provider(self):
if not TEST_MSI_AUTH:
print(" *** Skipped MSI Provider Test ***")
return
user_msi_object_id = os.environ.get("MSI_OBJECT_ID")
user_msi_client_id = os.environ.get("MSI_CLIENT_ID")
# system MSI
provider = MsiTokenProvider(KUSTO_URI, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
if user_msi_object_id is not None:
args = {"object_id": user_msi_object_id}
provider = MsiTokenProvider(KUSTO_URI, args, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
else:
print(" *** Skipped MSI Provider Client Id Test ***")
if user_msi_client_id is not None:
args = {"client_id": user_msi_client_id}
provider = MsiTokenProvider(KUSTO_URI, args, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
else:
print(" *** Skipped MSI Provider Object Id Test ***")
@aio_documented_by(TokenProviderTests.test_user_pass_provider)
@pytest.mark.asyncio
async def test_user_pass_provider(self):
username = os.environ.get("USER_NAME")
password = os.environ.get("USER_PASS")
auth = os.environ.get("USER_AUTH_ID", "organizations")
if username and password and auth:
provider = UserPassTokenProvider(KUSTO_URI, auth, username, password, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert self.get_token_value(token) is not None
else:
print(" *** Skipped User & Pass Provider Test ***")
@aio_documented_by(TokenProviderTests.test_device_auth_provider)
@pytest.mark.asyncio
async def test_device_auth_provider(self):
if not TEST_DEVICE_AUTH:
print(" *** Skipped User Device Flow Test ***")
return
def callback(x):
# break here if you debug this test, and get the code from 'x'
print(x)
provider = DeviceLoginTokenProvider(KUSTO_URI, "organizations", callback, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert self.get_token_value(token) is not None
@aio_documented_by(TokenProviderTests.test_app_key_provider)
@pytest.mark.asyncio
async def test_app_key_provider(self):
# default details are for kusto-client-e2e-test-app
# to run the test, get the key from Azure portal
app_id = os.environ.get("APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f")
auth_id = os.environ.get("APP_AUTH_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47")
app_key = os.environ.get("APP_KEY")
if app_id and app_key and auth_id:
provider = ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert self.get_token_value(token) is not None
else:
print(" *** Skipped App Id & Key Provider Test ***")
@aio_documented_by(TokenProviderTests.test_app_cert_provider)
@pytest.mark.asyncio
async def test_app_cert_provider(self):
# default details are for kusto-client-e2e-test-app
# to run the test download the certs from Azure Portal
cert_app_id = os.environ.get("CERT_APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f")
cert_auth = os.environ.get("CERT_AUTH", "72f988bf-86f1-41af-91ab-2d7cd011db47")
thumbprint = os.environ.get("CERT_THUMBPRINT")
public_cert_path = os.environ.get("PUBLIC_CERT_PATH")
pem_key_path = os.environ.get("CERT_PEM_KEY_PATH")
if pem_key_path and thumbprint and cert_app_id:
with open(pem_key_path, "rb") as file:
pem_key = file.read()
provider = ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert self.get_token_value(token) is not None
if public_cert_path:
with open(public_cert_path, "r") as file:
public_cert = file.read()
provider = ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint, public_cert, is_async=True)
token = await provider.get_token_async()
assert self.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert self.get_token_value(token) is not None
else:
print(" *** Skipped App Cert SNI Provider Test ***")
else:
print(" *** Skipped App Cert Provider Test ***")
@aio_documented_by(TokenProviderTests.test_cloud_mfa_off)
@pytest.mark.asyncio
async def test_cloud_mfa_off(self):
FAKE_URI = "https://fake_cluster_for_login_mfa_test.kusto.windows.net"
cloud = CloudInfo(
login_endpoint="https://login_endpoint",
login_mfa_required=False,
kusto_client_app_id="1234",
kusto_client_redirect_uri="",
kusto_service_resource_id="https://fakeurl.kusto.windows.net",
first_party_authority_url="",
)
CloudSettings._cloud_cache[FAKE_URI] = cloud
authority = "auth_test"
provider = UserPassTokenProvider(FAKE_URI, authority, "a", "b", is_async=True)
await provider._init_once_async(init_only_resources=True)
context = await provider.context_async()
assert context["authority"] == "https://login_endpoint/auth_test"
assert context["client_id"] == "1234"
assert provider._scopes == ["https://fakeurl.kusto.windows.net/.default"]
@aio_documented_by(TokenProviderTests.test_cloud_mfa_off)
@pytest.mark.asyncio
async def test_cloud_mfa_on(self):
FAKE_URI = "https://fake_cluster_for_login_mfa_test.kusto.windows.net"
cloud = CloudInfo(
login_endpoint="https://login_endpoint",
login_mfa_required=True,
kusto_client_app_id="1234",
kusto_client_redirect_uri="",
kusto_service_resource_id="https://fakeurl.kusto.windows.net",
first_party_authority_url="",
)
CloudSettings._cloud_cache[FAKE_URI] = cloud
authority = "auth_test"
provider = UserPassTokenProvider(FAKE_URI, authority, "a", "b", is_async=True)
await provider._init_once_async(init_only_resources=True)
context = await provider.context_async()
assert context["authority"] == "https://login_endpoint/auth_test"
assert context["client_id"] == "1234"
assert provider._scopes == ["https://fakeurl.kustomfa.windows.net/.default"]
def test_async_lock(self):
"""
This test makes sure that the lock inside of a TokenProvider, is created within the correct event loop.
Before this, the Lock was created once per class.
This meant that if someone created a new event loop, and created a provider in it, awaiting on the lock would cause an exception because it belongs to
a different loop.
Now the lock is instantiated for every class instance, avoiding this issue.
"""
async def start():
async def inner():
await asyncio.sleep(0.1)
return ""
provider = CallbackTokenProvider(token_callback=None, async_token_callback=inner, is_async=True)
await asyncio.gather(provider.get_token_async(), provider.get_token_async(), provider.get_token_async())
loop = asyncio.events.new_event_loop()
asyncio.events.set_event_loop(loop)
loop.run_until_complete(start())
azure-kusto-python-3.0.1/azure-kusto-data/tests/aio/test_kusto_client.py 0000664 0000000 0000000 00000020016 14172227630 0026554 0 ustar 00root root 0000000 0000000 """Tests for KustoClient."""
import json
import sys
from unittest.mock import patch
import pytest
from azure.kusto.data._cloud_settings import CloudSettings
from azure.kusto.data._decorators import aio_documented_by
from azure.kusto.data.client import ClientRequestProperties
from azure.kusto.data.exceptions import KustoMultiApiError
from azure.kusto.data.helpers import dataframe_from_result_table
from ..kusto_client_common import KustoClientTestsMixin, mocked_requests_post, proxy_kcsb
from ..test_kusto_client import TestKustoClient as KustoClientTestsSync
PANDAS = False
try:
import pandas
PANDAS = True
except:
pass
run_aio_tests = False
try:
from azure.kusto.data.aio.client import KustoClient
from aioresponses import aioresponses, CallbackResult
run_aio_tests = True
except:
pass
if sys.version_info < (3, 6):
run_aio_tests = False
@pytest.mark.skipif(not run_aio_tests, reason="requires aio")
@aio_documented_by(KustoClientTestsSync)
class TestKustoClient(KustoClientTestsMixin):
@staticmethod
def _mock_callback(url, **kwargs):
body = json.dumps(mocked_requests_post(str(url), **kwargs).json())
return CallbackResult(status=200, body=body)
def _mock_query(self, aioresponses_mock):
url = "{host}/v2/rest/query".format(host=self.HOST)
aioresponses_mock.post(url, callback=self._mock_callback)
def _mock_mgmt(self, aioresponses_mock):
url = "{host}/v1/rest/mgmt".format(host=self.HOST)
aioresponses_mock.post(url, callback=self._mock_callback)
def _mock_cloud_info(self, aioresponses_mock):
url = "{host}/v1/rest/auth/metadata".format(host=self.HOST)
aioresponses_mock.get(url, callback=self._mock_callback)
@aio_documented_by(KustoClientTestsSync.test_sanity_query)
@pytest.mark.asyncio
async def test_sanity_query(self):
with aioresponses() as aioresponses_mock:
self._mock_query(aioresponses_mock)
async with KustoClient(self.HOST) as client:
response = await client.execute_query("PythonTest", "Deft")
first_request = next(iter(aioresponses_mock.requests.values()))
self._assert_client_request_id(first_request[0].kwargs)
self._assert_sanity_query_response(response)
@aio_documented_by(KustoClientTestsSync.test_sanity_control_command)
@pytest.mark.asyncio
async def test_sanity_control_command(self):
with aioresponses() as aioresponses_mock:
self._mock_mgmt(aioresponses_mock)
async with KustoClient(self.HOST) as client:
response = await client.execute_mgmt("NetDefaultDB", ".show version")
self._assert_sanity_control_command_response(response)
@pytest.mark.skipif(not PANDAS, reason="requires pandas")
@aio_documented_by(KustoClientTestsSync.test_sanity_data_frame)
@pytest.mark.asyncio
async def test_sanity_data_frame(self):
with aioresponses() as aioresponses_mock:
self._mock_query(aioresponses_mock)
async with KustoClient(self.HOST) as client:
response = await client.execute_query("PythonTest", "Deft")
data_frame = dataframe_from_result_table(response.primary_results[0])
self._assert_sanity_data_frame_response(data_frame)
@aio_documented_by(KustoClientTestsSync.test_partial_results)
@pytest.mark.asyncio
async def test_partial_results(self):
async with KustoClient(self.HOST) as client:
query = """set truncationmaxrecords = 5;
range x from 1 to 10 step 1"""
properties = ClientRequestProperties()
properties.set_option(ClientRequestProperties.results_defer_partial_query_failures_option_name, False)
with aioresponses() as aioresponses_mock:
self._mock_query(aioresponses_mock)
with pytest.raises(KustoMultiApiError) as e:
await client.execute_query("PythonTest", query, properties)
errors = e.value.get_api_errors()
assert len(errors) == 1
assert errors[0].code == "LimitsExceeded"
properties.set_option(ClientRequestProperties.results_defer_partial_query_failures_option_name, True)
self._mock_query(aioresponses_mock)
response = await client.execute_query("PythonTest", query, properties)
self._assert_partial_results_response(response)
@aio_documented_by(KustoClientTestsSync.test_admin_then_query)
@pytest.mark.asyncio
async def test_admin_then_query(self):
with aioresponses() as aioresponses_mock:
self._mock_mgmt(aioresponses_mock)
async with KustoClient(self.HOST) as client:
query = ".show tables | project DatabaseName, TableName"
response = await client.execute_mgmt("PythonTest", query)
self._assert_admin_then_query_response(response)
@aio_documented_by(KustoClientTestsSync.test_dynamic)
@pytest.mark.asyncio
async def test_dynamic(self):
with aioresponses() as aioresponses_mock:
self._mock_query(aioresponses_mock)
async with KustoClient(self.HOST) as client:
query = """print dynamic(123), dynamic("123"), dynamic("test bad json"),"""
""" dynamic(null), dynamic('{"rowId":2,"arr":[0,2]}'), dynamic({"rowId":2,"arr":[0,2]})"""
response = await client.execute_query("PythonTest", query)
row = response.primary_results[0].rows[0]
self._assert_dynamic_response(row)
@aio_documented_by(KustoClientTestsSync.test_empty_result)
@pytest.mark.asyncio
async def test_empty_result(self):
with aioresponses() as aioresponses_mock:
self._mock_query(aioresponses_mock)
async with KustoClient(self.HOST) as client:
query = """print 'a' | take 0"""
response = await client.execute_query("PythonTest", query)
assert response.primary_results[0]
@aio_documented_by(KustoClientTestsSync.test_null_values_in_data)
@pytest.mark.asyncio
async def test_null_values_in_data(self):
with aioresponses() as aioresponses_mock:
self._mock_query(aioresponses_mock)
async with KustoClient(self.HOST) as client:
query = "PrimaryResultName"
response = await client.execute_query("PythonTest", query)
assert response is not None
@aio_documented_by(KustoClientTestsSync.test_sanity_query)
@pytest.mark.asyncio
async def test_request_id(self):
with aioresponses() as aioresponses_mock:
properties = ClientRequestProperties()
request_id = "test_request_id"
properties.client_request_id = request_id
self._mock_query(aioresponses_mock)
async with KustoClient(self.HOST) as client:
response = await client.execute_query("PythonTest", "Deft", properties=properties)
first_request = next(iter(aioresponses_mock.requests.values()))
self._assert_client_request_id(first_request[0].kwargs, value=request_id)
self._assert_sanity_query_response(response)
@aio_documented_by(KustoClientTestsSync.test_proxy_token_providers)
@pytest.mark.asyncio
async def test_proxy_token_providers(self, proxy_kcsb):
"""Test query V2."""
proxy = "https://my_proxy.sample"
kcsb, auth_supports_proxy = proxy_kcsb
async with KustoClient(kcsb) as client:
client.set_proxy(proxy)
assert client._proxy_url == proxy
expected_dict = {"http": proxy, "https": proxy}
if not auth_supports_proxy:
return
assert client._aad_helper.token_provider._proxy_dict == expected_dict
CloudSettings._cloud_cache.clear()
with patch("requests.get", side_effect=mocked_requests_post) as mock_get:
client._aad_helper.token_provider._init_resources()
mock_get.assert_called_with("https://somecluster.kusto.windows.net/v1/rest/auth/metadata", proxies=expected_dict)
azure-kusto-python-3.0.1/azure-kusto-data/tests/input/ 0000775 0000000 0000000 00000000000 14172227630 0023030 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-data/tests/input/adminthenquery.json 0000664 0000000 0000000 00000006031 14172227630 0026760 0 ustar 00root root 0000000 0000000 {
"Tables": [{
"TableName": "Table_0",
"Columns": [{
"ColumnName": "DatabaseName",
"DataType": "String"
},
{
"ColumnName": "TableName",
"DataType": "String"
}],
"Rows": [["Kuskus",
"KustoLogs"],
["Kuskus",
"LiorTmp"]]
},
{
"TableName": "Table_1",
"Columns": [{
"ColumnName": "Value",
"DataType": "String"
}],
"Rows": [["{\"Visualization\": null,\"Title\": null,\"XColumn\": null,\"Series\": null,\"YColumns\": null,\"XTitle\": null,\"YTitle\": null,\"XAxis\": null,\"YAxis\": null,\"Legend\": null,\"YSplit\": null,\"Accumulate\": false,\"IsQuerySorted\": false,\"Kind\": null}"]]
},
{
"TableName": "Table_2",
"Columns": [{
"ColumnName": "Timestamp",
"DataType": "DateTime"
},
{
"ColumnName": "Severity",
"DataType": "Int32"
},
{
"ColumnName": "SeverityName",
"DataType": "String"
},
{
"ColumnName": "StatusCode",
"DataType": "Int32"
},
{
"ColumnName": "StatusDescription",
"DataType": "String"
},
{
"ColumnName": "Count",
"DataType": "Int32"
},
{
"ColumnName": "RequestId",
"DataType": "Guid"
},
{
"ColumnName": "ActivityId",
"DataType": "Guid"
},
{
"ColumnName": "SubActivityId",
"DataType": "Guid"
},
{
"ColumnName": "ClientActivityId",
"DataType": "String"
}],
"Rows": [["2018-08-12T09:13:19.5200972Z",
4,
"Info",
0,
"Querycompletedsuccessfully",
1,
"b6651693-9325-41c8-a5bf-dcc21202cdf2",
"b6651693-9325-41c8-a5bf-dcc21202cdf2",
"1cfa59c2-7f29-4e58-aef8-590d4609818f",
"KPC.execute;721add30-f61a-44d0-ad89-812d06736260"],
["2018-08-12T09:13:19.5200972Z",
6,
"Stats",
0,
{
"ExecutionTime": 0.0,
"resource_usage": {
"cache": {
"memory": {
"hits": 0,
"misses": 0,
"total": 0
},
"disk": {
"hits": 0,
"misses": 0,
"total": 0
}
},
"cpu": {
"user": "00:00:00",
"kernel": "00:00:00",
"total cpu": "00:00:00"
},
"memory": {
"peak_per_node": 0
}
},
"input_dataset_statistics": {
"extents": {
"total": 0,
"scanned": 0
},
"rows": {
"total": 0,
"scanned": 0
}
},
"dataset_statistics": [{
"table_row_count": 2,
"table_size": 46
}]
},
1,
"b6651693-9325-41c8-a5bf-dcc21202cdf2",
"b6651693-9325-41c8-a5bf-dcc21202cdf2",
"1cfa59c2-7f29-4e58-aef8-590d4609818f",
"KPC.execute;721add30-f61a-44d0-ad89-812d06736260"]]
},
{
"TableName": "Table_3",
"Columns": [{
"ColumnName": "Ordinal",
"DataType": "Int64"
},
{
"ColumnName": "Kind",
"DataType": "String"
},
{
"ColumnName": "Name",
"DataType": "String"
},
{
"ColumnName": "Id",
"DataType": "String"
},
{
"ColumnName": "PrettyName",
"DataType": "String"
}],
"Rows": [[0,
"QueryResult",
"PrimaryResult",
"d6331ef2-d9f7-4d8c-8268-99f574babc82",
""],
[1,
"QueryProperties",
"@ExtendedProperties",
"876ccb1a-818e-431f-9147-6b72547cca3d",
""],
[2,
"QueryStatus",
"QueryStatus",
"00000000-0000-0000-0000-000000000000",
""]]
}]
}
azure-kusto-python-3.0.1/azure-kusto-data/tests/input/alternative_order.json 0000664 0000000 0000000 00000022233 14172227630 0027436 0 ustar 00root root 0000000 0000000 [
{
"FrameType": "DataSetHeader",
"Version": "v2.0",
"IsProgressive": false
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableName": "@ExtendedProperties",
"Columns": [
{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}
],
"TableKind": "QueryProperties",
"Rows": [
[
1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null}"
]
]
},
{
"FrameType": "DataTable",
"Columns": [
{
"ColumnName": "rownumber",
"ColumnType": "int"
},
{
"ColumnName": "rowguid",
"ColumnType": "string"
},
{
"ColumnName": "xdouble",
"ColumnType": "real"
},
{
"ColumnName": "xfloat",
"ColumnType": "real"
},
{
"ColumnName": "xbool",
"ColumnType": "bool"
},
{
"ColumnName": "xint16",
"ColumnType": "int"
},
{
"ColumnName": "xint32",
"ColumnType": "int"
},
{
"ColumnName": "xint64",
"ColumnType": "long"
},
{
"ColumnName": "xuint8",
"ColumnType": "long"
},
{
"ColumnName": "xuint16",
"ColumnType": "long"
},
{
"ColumnName": "xuint32",
"ColumnType": "long"
},
{
"ColumnName": "xuint64",
"ColumnType": "long"
},
{
"ColumnName": "xdate",
"ColumnType": "datetime"
},
{
"ColumnName": "xsmalltext",
"ColumnType": "string"
},
{
"ColumnName": "xtext",
"ColumnType": "string"
},
{
"ColumnName": "xnumberAsText",
"ColumnType": "string"
},
{
"ColumnName": "xtime",
"ColumnType": "timespan"
},
{
"ColumnName": "xtextWithNulls",
"ColumnType": "string"
},
{
"ColumnName": "xdynamicWithNulls",
"ColumnType": "dynamic"
}
],
"TableId": 1,
"TableKind": "PrimaryResult",
"TableName": "Deft",
"Rows": [
[
null,
"",
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
"",
"",
"",
null,
"",
""
],
[
0,
"00000000-0000-0000-0001-020304050607",
0.0,
0.0,
false,
0,
0,
0,
0,
0,
0,
0,
"2014-01-01T01:01:01.0000000Z",
"Zero",
"Zero",
"0",
"00:00:00",
"",
""
],
[
1,
"00000001-0000-0000-0001-020304050607",
1.0001,
1.01,
true,
1,
1,
1,
1,
1,
1,
1,
"2015-01-01T01:01:01.0000001Z",
"One",
"One",
"1",
"1.00:00:01.0010001",
"",
{
"rowId": 1,
"arr": [
0,
1
]
}
],
[
2,
"00000002-0000-0000-0001-020304050607",
2.0002,
2.02,
false,
2,
2,
2,
2,
2,
2,
2,
"2016-01-01T01:01:01.0000002Z",
"Two",
"Two",
"2",
"-2.00:00:02.0020002",
"",
{
"rowId": 2,
"arr": [
0,
2
]
}
],
[
3,
"00000003-0000-0000-0001-020304050607",
3.0003,
3.03,
true,
3,
3,
3,
3,
3,
3,
3,
"2017-01-01T01:01:01.0000003Z",
"Three",
"Three",
"3",
"3.00:00:03.0030003",
"",
{
"rowId": 3,
"arr": [
0,
3
]
}
],
[
4,
"00000004-0000-0000-0001-020304050607",
4.0004,
4.04,
false,
4,
4,
4,
4,
4,
4,
4,
"2018-01-01T01:01:01.0000004Z",
"Four",
"Four",
"4",
"-4.00:00:04.0040004",
"",
{
"rowId": 4,
"arr": [
0,
4
]
}
],
[
5,
"00000005-0000-0000-0001-020304050607",
5.0005,
5.05,
true,
5,
5,
5,
5,
5,
5,
5,
"2019-01-01T01:01:01.0000005Z",
"Five",
"Five",
"5",
"5.00:00:05.0050005",
"",
{
"rowId": 5,
"arr": [
0,
5
]
}
],
[
6,
"00000006-0000-0000-0001-020304050607",
6.0006,
6.06,
false,
6,
6,
6,
6,
6,
6,
6,
"2020-01-01T01:01:01.0000006Z",
"Six",
"Six",
"6",
"-6.00:00:06.0060006",
"",
{
"rowId": 6,
"arr": [
0,
6
]
}
],
[
7,
"00000007-0000-0000-0001-020304050607",
7.0007,
7.07,
true,
7,
7,
7,
7,
7,
7,
7,
"2021-01-01T01:01:01.0000007Z",
"Seven",
"Seven",
"7",
"7.00:00:07.0070007",
"",
{
"rowId": 7,
"arr": [
0,
7
]
}
],
[
8,
"00000008-0000-0000-0001-020304050607",
8.0008,
8.08,
false,
8,
8,
8,
8,
8,
8,
8,
"2022-01-01T01:01:01.0000008Z",
"Eight",
"Eight",
"8",
"-8.00:00:08.0080008",
"",
{
"rowId": 8,
"arr": [
0,
8
]
}
],
[
9,
"00000009-0000-0000-0001-020304050607",
9.0009,
9.09,
true,
9,
9,
9,
9,
9,
9,
9,
"2023-01-01T01:01:01.0000009Z",
"Nine",
"Nine",
"9",
"9.00:00:09.0090009",
"",
{
"rowId": 9,
"arr": [
0,
9
]
}
]
]
},
{
"FrameType": "DataTable",
"Columns": [
{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}
],
"TableId": 2,
"TableName": "QueryCompletionInformation",
"TableKind": "QueryCompletionInformation",
"Rows": [
[
"2018-04-30T12:25:11.0778067Z",
"unspecified;cc5ee2b9-9b77-4509-9a61-84ec8f0159c2",
"eeac049e-8a7d-4188-b797-6b5f2c9f9526",
"c6fb9714-5183-4092-8a02-825bd7aa1aee",
"df0aaeb0-4c8a-4d77-bc77-f714a4484a6b",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\"Count\":1,\"Text\":\"Query completed successfully\"}"
],
[
"2018-04-30T12:25:11.0778067Z",
"unspecified;cc5ee2b9-9b77-4509-9a61-84ec8f0159c2",
"eeac049e-8a7d-4188-b797-6b5f2c9f9526",
"c6fb9714-5183-4092-8a02-825bd7aa1aee",
"df0aaeb0-4c8a-4d77-bc77-f714a4484a6b",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.0156154,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":40,\"misses\":0,\"total\":40},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0}},\"cpu\":{\"user\":\"00:00:00\",\"kernel\":\"00:00:00\",\"total cpu\":\"00:00:00\"},\"memory\":{\"peak_per_node\":0}},\"dataset_statistics\":[{\"table_row_count\":11,\"table_size\":2444}]}"
]
]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}
] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/dataframe.json 0000664 0000000 0000000 00000010345 14172227630 0025652 0 ustar 00root root 0000000 0000000 [{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableName": "@ExtendedProperties",
"TableKind": "QueryProperties",
"Columns": [{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}],
"Rows": [[1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null}"]]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableName": "temp",
"TableKind": "PrimaryResult",
"Columns": [{
"ColumnName": "RecordName",
"ColumnType": "string"
},
{
"ColumnName": "RecordTime",
"ColumnType": "datetime"
},
{
"ColumnName": "RecordOffset",
"ColumnType": "timespan"
},
{
"ColumnName": "RecordBool",
"ColumnType": "bool"
},
{
"ColumnName": "RecordInt",
"ColumnType": "int"
},
{
"ColumnName": "RecordReal",
"ColumnType": "real"
}],
"Rows": [["now",
"2021-12-22T11:43:00Z",
0,
true,
5678,
3.14159],
["earliest datetime",
"0000-01-01T00:00:00Z",
0,
true,
5678,
NaN],
["latest datetime",
"9999-12-31T23:59:59Z",
0,
true,
5678,
Infinity],
["earliest pandas datetime",
"1677-09-21T00:12:44Z",
0,
true,
5678,
-Infinity],
["latest pandas datetime",
"2262-04-11T23:47:16Z",
0,
true,
5678,
3.14159],
["timedelta ticks",
"2021-12-22T11:43:00Z",
600000000,
true,
5678,
3.14159],
["timedelta string",
"2021-12-22T11:43:00Z",
"1.01:01:01.0",
true,
5678,
3.14159],
[null,
"",
0,
false,
0,
0]]
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableName": "QueryCompletionInformation",
"TableKind": "QueryCompletionInformation",
"Columns": [{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}],
"Rows": [["2018-05-01T09:32:38.916566Z",
"unspecified;e8e72755-786b-4bdc-835d-ea49d63d09fd",
"5935a050-e466-48a0-991d-0ec26bd61c7e",
"8182b177-7a80-4158-aca8-ff4fd8e7d3f8",
"6f3c1072-2739-461c-8aa7-3cfc8ff528a8",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\"Count\":1,\"Text\":\"Querycompletedsuccessfully\"}"],
["2018-05-01T09:32:38.916566Z",
"unspecified;e8e72755-786b-4bdc-835d-ea49d63d09fd",
"5935a050-e466-48a0-991d-0ec26bd61c7e",
"8182b177-7a80-4158-aca8-ff4fd8e7d3f8",
"6f3c1072-2739-461c-8aa7-3cfc8ff528a8",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.0156222,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":13,\"misses\":0,\"total\":13},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0}},\"cpu\":{\"user\":\"00: 00: 00\",\"kernel\":\"00: 00: 00\",\"totalcpu\":\"00: 00: 00\"},\"memory\":{\"peak_per_node\":16777312}},\"dataset_statistics\":[{\"table_row_count\":3,\"table_size\":191}]}"]]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/deft.json 0000664 0000000 0000000 00000027161 14172227630 0024654 0 ustar 00root root 0000000 0000000 [
{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableName": "@ExtendedProperties",
"TableKind": "QueryProperties",
"Columns": [
{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}
],
"Rows": [
[
1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null}"
]
]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableName": "Deft",
"TableKind": "PrimaryResult",
"Columns": [
{
"ColumnName": "rownumber",
"ColumnType": "int"
},
{
"ColumnName": "rowguid",
"ColumnType": "string"
},
{
"ColumnName": "xdouble",
"ColumnType": "real"
},
{
"ColumnName": "xfloat",
"ColumnType": "real"
},
{
"ColumnName": "xbool",
"ColumnType": "bool"
},
{
"ColumnName": "xint16",
"ColumnType": "int"
},
{
"ColumnName": "xint32",
"ColumnType": "int"
},
{
"ColumnName": "xint64",
"ColumnType": "long"
},
{
"ColumnName": "xuint8",
"ColumnType": "long"
},
{
"ColumnName": "xuint16",
"ColumnType": "long"
},
{
"ColumnName": "xuint32",
"ColumnType": "long"
},
{
"ColumnName": "xuint64",
"ColumnType": "long"
},
{
"ColumnName": "xdate",
"ColumnType": "datetime"
},
{
"ColumnName": "xsmalltext",
"ColumnType": "string"
},
{
"ColumnName": "xtext",
"ColumnType": "string"
},
{
"ColumnName": "xnumberAsText",
"ColumnType": "string"
},
{
"ColumnName": "xtime",
"ColumnType": "timespan"
},
{
"ColumnName": "xtextWithNulls",
"ColumnType": "string"
},
{
"ColumnName": "xdynamicWithNulls",
"ColumnType": "dynamic"
}
],
"Rows": [
[
null,
"",
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
"",
"",
"",
null,
"",
""
],
[
0,
"00000000-0000-0000-0001-020304050607",
0.0,
0.0,
false,
0,
0,
0,
0,
0,
0,
0,
"2014-01-01T01:01:01.0000000Z",
"Zero",
"Zero",
"0",
"00:00:00",
"",
""
],
[
1,
"00000001-0000-0000-0001-020304050607",
1.0001,
1.01,
true,
1,
1,
1,
1,
1,
1,
1,
"2015-01-01T01:01:01.0000001Z",
"One",
"One",
"1",
"1.00:00:01.0010001",
"",
{"rowId":1,"arr":[0,1]}
],
[
2,
"00000002-0000-0000-0001-020304050607",
2.0002,
2.02,
false,
2,
2,
2,
2,
2,
2,
2,
"2016-01-01T01:01:01.0000002Z",
"Two",
"Two",
"2",
"-2.00:00:02.0020002",
"",
{"rowId":2,"arr":[0,2]}
],
[
3,
"00000003-0000-0000-0001-020304050607",
3.0003,
3.03,
true,
3,
3,
3,
3,
3,
3,
3,
"2017-01-01T01:01:01.0000003Z",
"Three",
"Three",
"3",
"3.00:00:03.0030003",
"",
{"rowId":3,"arr":[0,3]}
],
[
4,
"00000004-0000-0000-0001-020304050607",
4.0004,
4.04,
false,
4,
4,
4,
4,
4,
4,
4,
"2018-01-01T01:01:01.0000004Z",
"Four",
"Four",
"4",
"-4.00:00:04.0040004",
"",
{"rowId":4,"arr":[0,4]}
],
[
5,
"00000005-0000-0000-0001-020304050607",
5.0005,
5.05,
true,
5,
5,
5,
5,
5,
5,
5,
"2019-01-01T01:01:01.0000005Z",
"Five",
"Five",
"5",
"5.00:00:05.0050005",
"",
{"rowId":5,"arr":[0,5]}
],
[
6,
"00000006-0000-0000-0001-020304050607",
6.0006,
6.06,
false,
6,
6,
6,
6,
6,
6,
6,
"2020-01-01T01:01:01.0000006Z",
"Six",
"Six",
"6",
"-6.00:00:06.0060006",
"",
{"rowId":6,"arr":[0,6]}
],
[
7,
"00000007-0000-0000-0001-020304050607",
7.0007,
7.07,
true,
7,
7,
7,
7,
7,
7,
7,
"2021-01-01T01:01:01.0000007Z",
"Seven",
"Seven",
"7",
"7.00:00:07.0070007",
"",
{"rowId":7,"arr":[0,7]}
],
[
8,
"00000008-0000-0000-0001-020304050607",
8.0008,
8.08,
false,
8,
8,
8,
8,
8,
8,
8,
"2022-01-01T01:01:01.0000008Z",
"Eight",
"Eight",
"8",
"-8.00:00:08.0080008",
"",
{"rowId":8,"arr":[0,8]}
],
[
9,
"00000009-0000-0000-0001-020304050607",
9.0009,
9.09,
true,
9,
9,
9,
9,
9,
9,
9,
"2023-01-01T01:01:01.0000009Z",
"Nine",
"Nine",
"9",
"9.00:00:09.0090009",
"",
{"rowId":9,"arr":[0,9]}
]
]
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableName": "QueryCompletionInformation",
"TableKind": "QueryCompletionInformation",
"Columns": [
{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}
],
"Rows": [
[
"2018-04-30T12:25:11.0778067Z",
"unspecified;cc5ee2b9-9b77-4509-9a61-84ec8f0159c2",
"eeac049e-8a7d-4188-b797-6b5f2c9f9526",
"c6fb9714-5183-4092-8a02-825bd7aa1aee",
"df0aaeb0-4c8a-4d77-bc77-f714a4484a6b",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\"Count\":1,\"Text\":\"Query completed successfully\"}"
],
[
"2018-04-30T12:25:11.0778067Z",
"unspecified;cc5ee2b9-9b77-4509-9a61-84ec8f0159c2",
"eeac049e-8a7d-4188-b797-6b5f2c9f9526",
"c6fb9714-5183-4092-8a02-825bd7aa1aee",
"df0aaeb0-4c8a-4d77-bc77-f714a4484a6b",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.0156154,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":40,\"misses\":0,\"total\":40},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0}},\"cpu\":{\"user\":\"00:00:00\",\"kernel\":\"00:00:00\",\"total cpu\":\"00:00:00\"},\"memory\":{\"peak_per_node\":0}},\"dataset_statistics\":[{\"table_row_count\":11,\"table_size\":2444}]}"
]
]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}
] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/deft_with_progressive_result.json 0000664 0000000 0000000 00000022361 14172227630 0031732 0 ustar 00root root 0000000 0000000 [
{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableName": "@ExtendedProperties",
"TableKind": "QueryProperties",
"Columns": [
{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}
],
"Rows": [
[
1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null}"
]
]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableName": "Deft",
"TableKind": "PrimaryResult",
"Columns": [
{
"ColumnName": "rownumber",
"ColumnType": "int"
},
{
"ColumnName": "rowguid",
"ColumnType": "string"
},
{
"ColumnName": "xdouble",
"ColumnType": "real"
},
{
"ColumnName": "xfloat",
"ColumnType": "real"
},
{
"ColumnName": "xbool",
"ColumnType": "bool"
},
{
"ColumnName": "xint16",
"ColumnType": "int"
},
{
"ColumnName": "xint32",
"ColumnType": "int"
},
{
"ColumnName": "xint64",
"ColumnType": "long"
},
{
"ColumnName": "xuint8",
"ColumnType": "long"
},
{
"ColumnName": "xuint16",
"ColumnType": "long"
},
{
"ColumnName": "xuint32",
"ColumnType": "long"
},
{
"ColumnName": "xuint64",
"ColumnType": "long"
},
{
"ColumnName": "xdate",
"ColumnType": "datetime"
},
{
"ColumnName": "xsmalltext",
"ColumnType": "string"
},
{
"ColumnName": "xtext",
"ColumnType": "string"
},
{
"ColumnName": "xnumberAsText",
"ColumnType": "string"
},
{
"ColumnName": "xtime",
"ColumnType": "timespan"
},
{
"ColumnName": "xtextWithNulls",
"ColumnType": "string"
},
{
"ColumnName": "xdynamicWithNulls",
"ColumnType": "dynamic"
}
],
"Rows": [
[
null,
"",
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
"",
"",
"",
null,
"",
""
],
[
0,
"00000000-0000-0000-0001-020304050607",
0.0,
0.0,
false,
0,
0,
0,
0,
0,
0,
0,
"2014-01-01T01:01:01.0000000Z",
"Zero",
"Zero",
"0",
"00:00:00",
"",
""
],
[
1,
"00000001-0000-0000-0001-020304050607",
1.0001,
1.01,
true,
1,
1,
1,
1,
1,
1,
1,
"2015-01-01T01:01:01.0000001Z",
"One",
"One",
"1",
"1.00:00:01.0010001",
"",
{
"rowId": 1,
"arr": [
0,
1
]
}
],
[
2,
"00000002-0000-0000-0001-020304050607",
2.0002,
2.02,
false,
2,
2,
2,
2,
2,
2,
2,
"2016-01-01T01:01:01.0000002Z",
"Two",
"Two",
"2",
"-2.00:00:02.0020002",
"",
{
"rowId": 2,
"arr": [
0,
2
]
}
],
[
3,
"00000003-0000-0000-0001-020304050607",
3.0003,
3.03,
true,
3,
3,
3,
3,
3,
3,
3,
"2017-01-01T01:01:01.0000003Z",
"Three",
"Three",
"3",
"3.00:00:03.0030003",
"",
{
"rowId": 3,
"arr": [
0,
3
]
}
],
[
4,
"00000004-0000-0000-0001-020304050607",
4.0004,
4.04,
false,
4,
4,
4,
4,
4,
4,
4,
"2018-01-01T01:01:01.0000004Z",
"Four",
"Four",
"4",
"-4.00:00:04.0040004",
"",
{
"rowId": 4,
"arr": [
0,
4
]
}
],
[
5,
"00000005-0000-0000-0001-020304050607",
5.0005,
5.05,
true,
5,
5,
5,
5,
5,
5,
5,
"2019-01-01T01:01:01.0000005Z",
"Five",
"Five",
"5",
"5.00:00:05.0050005",
"",
{
"rowId": 5,
"arr": [
0,
5
]
}
],
[
6,
"00000006-0000-0000-0001-020304050607",
6.0006,
6.06,
false,
6,
6,
6,
6,
6,
6,
6,
"2020-01-01T01:01:01.0000006Z",
"Six",
"Six",
"6",
"-6.00:00:06.0060006",
"",
{
"rowId": 6,
"arr": [
0,
6
]
}
],
[
7,
"00000007-0000-0000-0001-020304050607",
7.0007,
7.07,
true,
7,
7,
7,
7,
7,
7,
7,
"2021-01-01T01:01:01.0000007Z",
"Seven",
"Seven",
"7",
"7.00:00:07.0070007",
"",
{
"rowId": 7,
"arr": [
0,
7
]
}
],
[
8,
"00000008-0000-0000-0001-020304050607",
8.0008,
8.08,
false,
8,
8,
8,
8,
8,
8,
8,
"2022-01-01T01:01:01.0000008Z",
"Eight",
"Eight",
"8",
"-8.00:00:08.0080008",
"",
{
"rowId": 8,
"arr": [
0,
8
]
}
],
[
9,
"00000009-0000-0000-0001-020304050607",
9.0009,
9.09,
true,
9,
9,
9,
9,
9,
9,
9,
"2023-01-01T01:01:01.0000009Z",
"Nine",
"Nine",
"9",
"9.00:00:09.0090009",
"",
{
"rowId": 9,
"arr": [
0,
9
]
}
]
]
},
{
"FrameType": "TableProgress",
"TableId": 1,
"TableProgress": 0.0
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableName": "QueryCompletionInformation",
"TableKind": "QueryCompletionInformation",
"Columns": [
{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}
],
"Rows": [
[
"2018-04-30T12:25:11.0778067Z",
"unspecified;cc5ee2b9-9b77-4509-9a61-84ec8f0159c2",
"eeac049e-8a7d-4188-b797-6b5f2c9f9526",
"c6fb9714-5183-4092-8a02-825bd7aa1aee",
"df0aaeb0-4c8a-4d77-bc77-f714a4484a6b",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\"Count\":1,\"Text\":\"Query completed successfully\"}"
],
[
"2018-04-30T12:25:11.0778067Z",
"unspecified;cc5ee2b9-9b77-4509-9a61-84ec8f0159c2",
"eeac049e-8a7d-4188-b797-6b5f2c9f9526",
"c6fb9714-5183-4092-8a02-825bd7aa1aee",
"df0aaeb0-4c8a-4d77-bc77-f714a4484a6b",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.0156154,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":40,\"misses\":0,\"total\":40},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0}},\"cpu\":{\"user\":\"00:00:00\",\"kernel\":\"00:00:00\",\"total cpu\":\"00:00:00\"},\"memory\":{\"peak_per_node\":0}},\"dataset_statistics\":[{\"table_row_count\":11,\"table_size\":2444}]}"
]
]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}
] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/dynamic.json 0000664 0000000 0000000 00000006620 14172227630 0025353 0 ustar 00root root 0000000 0000000 [{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableKind": "QueryProperties",
"TableName": "@ExtendedProperties",
"Columns": [{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}],
"Rows": [[1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"AnomalyColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null,\"Ymin\":\"NaN\",\"Ymax\":\"NaN\"}"]]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableKind": "PrimaryResult",
"TableName": "PrimaryResult",
"Columns": [{
"ColumnName": "print_0",
"ColumnType": "dynamic"
},
{
"ColumnName": "print_1",
"ColumnType": "dynamic"
},
{
"ColumnName": "print_2",
"ColumnType": "dynamic"
},
{
"ColumnName": "print_3",
"ColumnType": "dynamic"
},
{
"ColumnName": "print_4",
"ColumnType": "dynamic"
},
{
"ColumnName": "print_5",
"ColumnType": "dynamic"
}],
"Rows": [[123,
"123",
"test bad json",
null,
"{\"rowId\":2,\"arr\":[0,2]}",
{
"rowId": 2,
"arr": [0,
2]
}]]
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableKind": "QueryCompletionInformation",
"TableName": "QueryCompletionInformation",
"Columns": [{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}],
"Rows": [["2019-02-10T12:07:01.0562684Z",
"KPC.execute;f0131f65-d1ed-4c9d-9110-ddf9879f7ff6",
"4400a720-f11d-43b6-9097-63462d35bcd4",
"bf1b857c-e857-44f6-8d8c-156081bfb92c",
"57e0f58d-398c-490b-b900-9f308c531af7",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\"Count\":1,\"Text\":\"Query completed successfully\"}"],
["2019-02-10T12:07:01.0562684Z",
"KPC.execute;f0131f65-d1ed-4c9d-9110-ddf9879f7ff6",
"4400a720-f11d-43b6-9097-63462d35bcd4",
"bf1b857c-e857-44f6-8d8c-156081bfb92c",
"57e0f58d-398c-490b-b900-9f308c531af7",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.1875076,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":0,\"misses\":0,\"total\":0},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0},\"shards\":{\"hitbytes\":0,\"missbytes\":0,\"bypassbytes\":0}},\"cpu\":{\"user\":\"00:00:00\",\"kernel\":\"00:00:00\",\"total cpu\":\"00:00:00\"},\"memory\":{\"peak_per_node\":0}},\"input_dataset_statistics\":{\"extents\":{\"total\":0,\"scanned\":0},\"rows\":{\"total\":0,\"scanned\":0},\"rowstores\":{\"scanned_rows\":0,\"scanned_values_size\":0}},\"dataset_statistics\":[{\"table_row_count\":1,\"table_size\":141}]}"]]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/null_values.json 0000664 0000000 0000000 00000006460 14172227630 0026262 0 ustar 00root root 0000000 0000000 [{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableKind": "QueryProperties",
"TableName": "@ExtendedProperties",
"Columns": [{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}],
"Rows": [[null,
null,
null]]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableKind": "PrimaryResult",
"TableName": "PrimaryResultName",
"Columns": [{
"ColumnName": "String",
"ColumnType": "string"
},
{
"ColumnName": "Int",
"ColumnType": "int"
},
{
"ColumnName": "Bool",
"ColumnType": "bool"
},
{
"ColumnName": "Datetime",
"ColumnType": "datetime"
},
{
"ColumnName": "Dynamic",
"ColumnType": "dynamic"
},
{
"ColumnName": "Guid",
"ColumnType": "guid"
},
{
"ColumnName": "Long",
"ColumnType": "long"
},
{
"ColumnName": "Real",
"ColumnType": "real"
},
{
"ColumnName": "Timespan",
"ColumnType": "timespan"
},
{
"ColumnName": "Decimal",
"ColumnType": "decimal"
}],
"Rows": [[null,null,null,null,null,null,null,null,null,null]]
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableKind": "QueryCompletionInformation",
"TableName": "QueryCompletionInformation",
"Columns": [{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}],
"Rows": [["2019-02-12T10:23:02.0413963Z",
"KPC.execute;57050c90-8a7d-4b29-b8d0-a40688a8185c",
"dfbaa865-e29d-46e0-af17-be22c6c113ac",
"e2bf7a6c-adf1-48da-b117-667a92874d38",
"81409d63-718b-4d06-9711-7eab476b7ceb",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\"Count\":1,\"Text\":\"Querycompletedsuccessfully\"}"],
["2019-02-12T10:23:02.0413963Z",
"KPC.execute;57050c90-8a7d-4b29-b8d0-a40688a8185c",
"dfbaa865-e29d-46e0-af17-be22c6c113ac",
"e2bf7a6c-adf1-48da-b117-667a92874d38",
"81409d63-718b-4d06-9711-7eab476b7ceb",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.0156475,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":0,\"misses\":0,\"total\":0},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0},\"shards\":{\"hitbytes\":0,\"missbytes\":0,\"bypassbytes\":0}},\"cpu\":{\"user\":\"00: 00: 00\",\"kernel\":\"00: 00: 00\",\"totalcpu\":\"00: 00: 00\"},\"memory\":{\"peak_per_node\":0}},\"input_dataset_statistics\":{\"extents\":{\"total\":0,\"scanned\":0},\"rows\":{\"total\":0,\"scanned\":0},\"rowstores\":{\"scanned_rows\":0,\"scanned_values_size\":0}},\"dataset_statistics\":[{\"table_row_count\":0,\"table_size\":0}]}"]]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/progressive_result.json 0000664 0000000 0000000 00000026251 14172227630 0027677 0 ustar 00root root 0000000 0000000 [
{
"FrameType": "DataSetHeader",
"IsProgressive": true,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableKind": "QueryProperties",
"TableName": "@ExtendedProperties",
"Columns": [
{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}
],
"Rows": [
[
1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"AnomalyColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null,\"Ymin\":\"NaN\",\"Ymax\":\"NaN\"}"
]
]
},
{
"FrameType": "TableHeader",
"TableId": 1,
"TableKind": "PrimaryResult",
"TableName": "PrimaryResult",
"Columns": [
{
"ColumnName": "StartTime",
"ColumnType": "datetime"
},
{
"ColumnName": "EndTime",
"ColumnType": "datetime"
},
{
"ColumnName": "EpisodeId",
"ColumnType": "int"
},
{
"ColumnName": "EventId",
"ColumnType": "int"
},
{
"ColumnName": "State",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "string"
},
{
"ColumnName": "InjuriesDirect",
"ColumnType": "int"
},
{
"ColumnName": "InjuriesIndirect",
"ColumnType": "int"
},
{
"ColumnName": "DeathsDirect",
"ColumnType": "int"
},
{
"ColumnName": "DeathsIndirect",
"ColumnType": "int"
},
{
"ColumnName": "DamageProperty",
"ColumnType": "int"
},
{
"ColumnName": "DamageCrops",
"ColumnType": "int"
},
{
"ColumnName": "Source",
"ColumnType": "string"
},
{
"ColumnName": "BeginLocation",
"ColumnType": "string"
},
{
"ColumnName": "EndLocation",
"ColumnType": "string"
},
{
"ColumnName": "BeginLat",
"ColumnType": "real"
},
{
"ColumnName": "BeginLon",
"ColumnType": "real"
},
{
"ColumnName": "EndLat",
"ColumnType": "real"
},
{
"ColumnName": "EndLon",
"ColumnType": "real"
},
{
"ColumnName": "EpisodeNarrative",
"ColumnType": "string"
},
{
"ColumnName": "EventNarrative",
"ColumnType": "string"
},
{
"ColumnName": "StormSummary",
"ColumnType": "dynamic"
}
]
},
{
"FrameType": "TableFragment",
"TableFragmentType": "DataAppend",
"TableId": 1,
"Rows": [
[
"2007-01-01T00:00:00Z",
"2007-01-01T00:00:00Z",
2592,
13208,
"NORTH CAROLINA",
"Thunderstorm Wind",
0,
0,
0,
0,
0,
0,
"Public",
"CASAR",
"CASAR",
35.52,
-81.63,
35.52,
-81.63,
"A small cluster of thunderstorms moved rapidly across the foothills and piedmont of western North Carolina, producing scattered wind damage.",
"Several trees down.",
{
"TotalDamages": 0,
"StartTime": "2007-01-01T00:00:00.0000000Z",
"EndTime": "2007-01-01T00:00:00.0000000Z",
"Details": {
"Description": "Several trees down.",
"Location": "NORTH CAROLINA"
}
}
],
[
"2007-01-01T00:00:00Z",
"2007-01-01T05:00:00Z",
4171,
23358,
"WISCONSIN",
"Winter Storm",
0,
0,
0,
0,
0,
0,
"COOP Observer",
"",
"",
null,
null,
null,
null,
"A powerful storm system moved from the southern plains through eastern Wisconsin, bringing copious amounts of precipitation to northwest Wisconsin from December 30th through the morning of January 1st. Deep tropical moisture out ahead of the storm brought heavy rains to the area on December 30th and 31st. Many areas saw over 1 inch of rain. Cold air wrapped around the system and changed the rain over to snow New Year's Eve. The snow finally ended during the early morning hours of New Year's Day, with the heaviest amounts of 6 to 12 inches reported across Ashland and Iron counties.",
"",
{
"TotalDamages": 0,
"StartTime": "2007-01-01T00:00:00.0000000Z",
"EndTime": "2007-01-01T05:00:00.0000000Z",
"Details": {
"Description": "",
"Location": "WISCONSIN"
}
}
],
[
"2007-01-01T00:00:00Z",
"2007-01-01T05:00:00Z",
4171,
23357,
"WISCONSIN",
"Winter Storm",
0,
0,
0,
0,
0,
0,
"COOP Observer",
"",
"",
null,
null,
null,
null,
"A powerful storm system moved from the southern plains through eastern Wisconsin, bringing copious amounts of precipitation to northwest Wisconsin from December 30th through the morning of January 1st. Deep tropical moisture out ahead of the storm brought heavy rains to the area on December 30th and 31st. Many areas saw over 1 inch of rain. Cold air wrapped around the system and changed the rain over to snow New Year's Eve. The snow finally ended during the early morning hours of New Year's Day, with the heaviest amounts of 6 to 12 inches reported across Ashland and Iron counties.",
"",
{
"TotalDamages": 0,
"StartTime": "2007-01-01T00:00:00.0000000Z",
"EndTime": "2007-01-01T05:00:00.0000000Z",
"Details": {
"Description": "",
"Location": "WISCONSIN"
}
}
],
[
"2007-01-01T00:00:00Z",
"2007-01-01T06:00:00Z",
1930,
9494,
"NEW YORK",
"Winter Weather",
0,
0,
0,
0,
2000,
0,
"Department of Highways",
"",
"",
null,
null,
null,
null,
"A weak area of low pressure moved across Ontario and Quebec provinces in Canada during the morning and afternoon of the 1st. Mild, moist air traveled over a seasonably cool airmass across New York and this resulted in a period of freezing rain from around Midnight to just before dawn. Freezing rain accumulated up to 1/4 of an inch across northern New York resulting in slick roads and several vehicle accidents.",
"",
{
"TotalDamages": 2000,
"StartTime": "2007-01-01T00:00:00.0000000Z",
"EndTime": "2007-01-01T06:00:00.0000000Z",
"Details": {
"Description": "",
"Location": "NEW YORK"
}
}
],
[
"2007-01-01T00:00:00Z",
"2007-01-01T06:00:00Z",
1930,
9488,
"NEW YORK",
"Winter Weather",
0,
0,
0,
0,
2000,
0,
"Department of Highways",
"",
"",
null,
null,
null,
null,
"A weak area of low pressure moved across Ontario and Quebec provinces in Canada during the morning and afternoon of the 1st. Mild, moist air traveled over a seasonably cool airmass across New York and this resulted in a period of freezing rain from around Midnight to just before dawn. Freezing rain accumulated up to 1/4 of an inch across northern New York resulting in slick roads and several vehicle accidents.",
"",
{
"TotalDamages": 2000,
"StartTime": "2007-01-01T00:00:00.0000000Z",
"EndTime": "2007-01-01T06:00:00.0000000Z",
"Details": {
"Description": "",
"Location": "NEW YORK"
}
}
]
]
},
{
"FrameType": "TableProgress",
"TableId": 1,
"TableProgress": 0.0
},
{
"FrameType": "TableCompletion",
"TableId": 1,
"RowCount": 5
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableKind": "QueryCompletionInformation",
"TableName": "QueryCompletionInformation",
"Columns": [
{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}
],
"Rows": [
[
"2021-10-17T08:28:04.3361959Z",
"KPC.execute;e6ab4da1-748a-4d7e-ae35-c8b9de57d5fc",
"d26c0e33-1a52-493e-931b-c3c37e4adb70",
"d6e77cac-8a3c-4eff-aa3a-85b7fa59c724",
"cd40ac74-c68c-4afa-aad1-992d599bc681",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\"Count\":1,\"Text\":\"Query completed successfully\"}"
],
[
"2021-10-17T08:28:04.3361959Z",
"KPC.execute;e6ab4da1-748a-4d7e-ae35-c8b9de57d5fc",
"d26c0e33-1a52-493e-931b-c3c37e4adb70",
"d6e77cac-8a3c-4eff-aa3a-85b7fa59c724",
"cd40ac74-c68c-4afa-aad1-992d599bc681",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.1250077,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":0,\"misses\":0,\"total\":0},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0},\"shards\":{\"hot\":{\"hitbytes\":521712,\"missbytes\":0,\"retrievebytes\":0},\"cold\":{\"hitbytes\":0,\"missbytes\":0,\"retrievebytes\":0},\"bypassbytes\":0}},\"cpu\":{\"user\":\"00:00:00\",\"kernel\":\"00:00:00\",\"total cpu\":\"00:00:00\"},\"memory\":{\"peak_per_node\":1179040},\"network\":{\"inter_cluster_total_bytes\":5072,\"intra_cluster_total_bytes\":0}},\"input_dataset_statistics\":{\"extents\":{\"total\":1,\"scanned\":1,\"scanned_min_datetime\":\"2021-10-17T05:47:23.0551402Z\",\"scanned_max_datetime\":\"2021-10-17T05:47:23.0551402Z\"},\"rows\":{\"total\":59066,\"scanned\":5},\"rowstores\":{\"scanned_rows\":0,\"scanned_values_size\":0},\"shards\":{\"queries_generic\":1,\"queries_specialized\":0}},\"dataset_statistics\":[{\"table_row_count\":5,\"table_size\":3594}]}"
]
]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}
] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/query_partial_results_defer_is_false.json 0000664 0000000 0000000 00000006035 14172227630 0033403 0 ustar 00root root 0000000 0000000 [{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableKind": "QueryProperties",
"TableName": "@ExtendedProperties",
"Columns": [{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}],
"Rows": [[1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"AnomalyColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null}"]]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableKind": "PrimaryResult",
"TableName": "PrimaryResult",
"Columns": [{
"ColumnName": "x",
"ColumnType": "long"
}],
"Rows": [[1],
[2],
[3],
[4],
[5],
{
"OneApiErrors": [{
"error": {
"code": "LimitsExceeded",
"message": "Request is invalid and cannot be executed.",
"@type": "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
"@message": "Query execution has exceeded the allowed limits (80DA0003): .",
"@context": {
"timestamp": "2018-12-10T15:10:48.8352222Z",
"machineName": "RD0003FFBEDEB9",
"processName": "Kusto.Azure.Svc",
"processId": 4328,
"threadId": 7284,
"appDomainName": "RdRuntime",
"clientRequestd": "KPC.execute;d3a43e37-0d7f-47a9-b6cd-a889b2aee3d3",
"activityId": "a57ec272-8846-49e6-b458-460b841ed47d",
"subActivityId": "a57ec272-8846-49e6-b458-460b841ed47d",
"activityType": "PO-OWIN-CallContext",
"parentActivityId": "a57ec272-8846-49e6-b458-460b841ed47d",
"activityStack": "(Activity stack: CRID=KPC.execute;d3a43e37-0d7f-47a9-b6cd-a889b2aee3d3 ARID=a57ec272-8846-49e6-b458-460b841ed47d > PO-OWIN-CallContext/a57ec272-8846-49e6-b458-460b841ed47d)"
},
"@permanent": false
}
}]
}]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": true,
"Cancelled": false,
"OneApiErrors": [{
"error": {
"code": "LimitsExceeded",
"message": "Request is invalid and cannot be executed.",
"@type": "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
"@message": "Query execution has exceeded the allowed limits (80DA0003): .",
"@context": {
"timestamp": "2018-12-10T15:10:48.8352222Z",
"machineName": "RD0003FFBEDEB9",
"processName": "Kusto.Azure.Svc",
"processId": 4328,
"threadId": 7284,
"appDomainName": "RdRuntime",
"clientRequestd": "KPC.execute;d3a43e37-0d7f-47a9-b6cd-a889b2aee3d3",
"activityId": "a57ec272-8846-49e6-b458-460b841ed47d",
"subActivityId": "a57ec272-8846-49e6-b458-460b841ed47d",
"activityType": "PO-OWIN-CallContext",
"parentActivityId": "a57ec272-8846-49e6-b458-460b841ed47d",
"activityStack": "(Activity stack: CRID=KPC.execute;d3a43e37-0d7f-47a9-b6cd-a889b2aee3d3 ARID=a57ec272-8846-49e6-b458-460b841ed47d > PO-OWIN-CallContext/a57ec272-8846-49e6-b458-460b841ed47d)"
},
"@permanent": false
}
}]
}] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/query_partial_results_defer_is_true.json 0000664 0000000 0000000 00000006135 14172227630 0033271 0 ustar 00root root 0000000 0000000 [{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableKind": "QueryProperties",
"TableName": "@ExtendedProperties",
"Columns": [{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}],
"Rows": [[1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"AnomalyColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null}"]]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableKind": "PrimaryResult",
"TableName": "PrimaryResult",
"Columns": [{
"ColumnName": "x",
"ColumnType": "long"
}],
"Rows": [[1],
[2],
[3],
[4],
[5]]
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableKind": "QueryCompletionInformation",
"TableName": "QueryCompletionInformation",
"Columns": [{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}],
"Rows": [["2018-12-10T15:12:14.2991851Z",
"KPC.execute;eba30dbd-9963-4c70-902d-60169b8b5a9c",
"031555a6-3a4f-4766-b3c3-fd28bf5d8ad6",
"a1e0d6fd-1def-46eb-b630-12efa5f88041",
"8421d4a0-d5b8-4e04-b345-019b2b2edf85",
2,
"Error",
-2133196797,
"Query result set too large (E_QUERY_RESULT_SET_TOO_LARGE). (-2133196797)",
1,
"QueryLimitsExceeded",
"{\"Count\":1,\"Text\":\"Queryresultsethasexceededtheinternalrecordcountlimit5(E_QUERY_RESULT_SET_TOO_LARGE;seehttp: //aka.ms/kustoquerylimits)\"}"],
["2018-12-10T15:12:14.2991851Z",
"KPC.execute;eba30dbd-9963-4c70-902d-60169b8b5a9c",
"031555a6-3a4f-4766-b3c3-fd28bf5d8ad6",
"a1e0d6fd-1def-46eb-b630-12efa5f88041",
"8421d4a0-d5b8-4e04-b345-019b2b2edf85",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.0,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":0,\"misses\":0,\"total\":0},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0}},\"cpu\":{\"user\":\"00: 00: 00\",\"kernel\":\"00: 00: 00\",\"totalcpu\":\"00: 00: 00\"},\"memory\":{\"peak_per_node\":16777312}},\"input_dataset_statistics\":{\"extents\":{\"total\":0,\"scanned\":0},\"rows\":{\"total\":0,\"scanned\":0},\"rowstores\":{\"scanned_rows\":0,\"scanned_values_size\":0}},\"dataset_statistics\":[{\"table_row_count\":5,\"table_size\":40}]}"]]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}] azure-kusto-python-3.0.1/azure-kusto-data/tests/input/versionshowcommandresult.json 0000664 0000000 0000000 00000000515 14172227630 0031110 0 ustar 00root root 0000000 0000000 {"Tables":[{"TableName":"Table_0","Columns":[{"ColumnName":"BuildVersion","DataType":"String"},{"ColumnName":"BuildTime","DataType":"DateTime"},{"ColumnName":"ServiceType","DataType":"String"},{"ColumnName":"ProductVersion","DataType":"String"}],"Rows":[["1.0.6693.14577","2018-04-29T08:05:54Z","Engine","KustoMain_2018.04.29.5"]]}]} azure-kusto-python-3.0.1/azure-kusto-data/tests/input/zero_results.json 0000664 0000000 0000000 00000005767 14172227630 0026502 0 ustar 00root root 0000000 0000000 [{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableKind": "QueryProperties",
"TableName": "@ExtendedProperties",
"Columns": [{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}],
"Rows": [[1,
"Visualization",
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"AnomalyColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null,\"Ymin\":\"NaN\",\"Ymax\":\"NaN\"}"]]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableKind": "PrimaryResult",
"TableName": "PrimaryResult",
"Columns": [{
"ColumnName": "print_0",
"ColumnType": "string"
}],
"Rows": []
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableKind": "QueryCompletionInformation",
"TableName": "QueryCompletionInformation",
"Columns": [{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}],
"Rows": [["2019-02-12T10:23:02.0413963Z",
"KPC.execute;57050c90-8a7d-4b29-b8d0-a40688a8185c",
"dfbaa865-e29d-46e0-af17-be22c6c113ac",
"e2bf7a6c-adf1-48da-b117-667a92874d38",
"81409d63-718b-4d06-9711-7eab476b7ceb",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\"Count\":1,\"Text\":\"Querycompletedsuccessfully\"}"],
["2019-02-12T10:23:02.0413963Z",
"KPC.execute;57050c90-8a7d-4b29-b8d0-a40688a8185c",
"dfbaa865-e29d-46e0-af17-be22c6c113ac",
"e2bf7a6c-adf1-48da-b117-667a92874d38",
"81409d63-718b-4d06-9711-7eab476b7ceb",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\"ExecutionTime\":0.0156475,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":0,\"misses\":0,\"total\":0},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0},\"shards\":{\"hitbytes\":0,\"missbytes\":0,\"bypassbytes\":0}},\"cpu\":{\"user\":\"00: 00: 00\",\"kernel\":\"00: 00: 00\",\"totalcpu\":\"00: 00: 00\"},\"memory\":{\"peak_per_node\":0}},\"input_dataset_statistics\":{\"extents\":{\"total\":0,\"scanned\":0},\"rows\":{\"total\":0,\"scanned\":0},\"rowstores\":{\"scanned_rows\":0,\"scanned_values_size\":0}},\"dataset_statistics\":[{\"table_row_count\":0,\"table_size\":0}]}"]]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}] azure-kusto-python-3.0.1/azure-kusto-data/tests/kusto_client_common.py 0000664 0000000 0000000 00000051320 14172227630 0026317 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import json
import os
import uuid
from datetime import datetime, timedelta
from io import StringIO
from typing import Optional, Any, Dict, Union, Iterator, Tuple
import pytest
from dateutil.tz import UTC
from requests import HTTPError
from azure.kusto.data import KustoConnectionStringBuilder
from azure.kusto.data._models import KustoResultRow, KustoResultTable, KustoStreamingResultTable
from azure.kusto.data.response import WellKnownDataSet, KustoStreamingResponseDataSet, KustoResponseDataSet
PANDAS = False
try:
import pandas
PANDAS = True
except:
pass
def mocked_requests_post(*args, **kwargs):
"""Mock to replace requests.Session.post"""
class Raw(StringIO):
def __init__(self, initial_value: Optional[str]) -> None:
super().__init__(initial_value, None)
self.decode_content = False
class MockResponse:
"""Mock class for KustoResponse."""
def __init__(self, json_data: Optional[Dict[str, Any]], status_code: int, url: str):
self.json_data = json_data
self.text = str(json_data)
self.status_code = status_code
self.headers = None
self.reason = ""
self.url = url
self.raw = Raw(json.dumps(json_data))
def json(self) -> Optional[Dict[str, Any]]:
"""Get json data from response."""
return self.json_data
def raise_for_status(self):
"""Raises stored :class:`HTTPError`, if one occurred."""
http_error_msg = ""
if isinstance(self.reason, bytes):
# We attempt to decode utf-8 first because some servers
# choose to localize their reason strings. If the string
# isn't utf-8, we fall back to iso-8859-1 for all other
# encodings. (See PR #3538)
try:
reason = self.reason.decode("utf-8")
except UnicodeDecodeError:
reason = self.reason.decode("iso-8859-1")
else:
reason = self.reason
if 400 <= self.status_code < 500:
http_error_msg = "%s Client Error: %s for url: %s" % (self.status_code, reason, self.url)
elif 500 <= self.status_code < 600:
http_error_msg = "%s Server Error: %s for url: %s" % (self.status_code, reason, self.url)
if http_error_msg:
raise HTTPError(http_error_msg, response=self)
url = args[0]
if url == "https://somecluster.kusto.windows.net/v2/rest/query":
if "truncationmaxrecords" in kwargs["json"]["csl"]:
if json.loads(kwargs["json"]["properties"])["Options"]["deferpartialqueryfailures"]:
file_name = "query_partial_results_defer_is_true.json"
else:
file_name = "query_partial_results_defer_is_false.json"
elif "Deft" in kwargs["json"]["csl"]:
file_name = "deft.json"
elif "print dynamic" in kwargs["json"]["csl"]:
file_name = "dynamic.json"
elif "take 0" in kwargs["json"]["csl"]:
file_name = "zero_results.json"
elif "PrimaryResultName" in kwargs["json"]["csl"]:
file_name = "null_values.json"
else:
raise Exception("Invalid file name")
with open(os.path.join(os.path.dirname(__file__), "input", file_name), "r") as response_file:
data = response_file.read()
return MockResponse(json.loads(data), 200, url)
elif url == "https://somecluster.kusto.windows.net/v1/rest/mgmt":
if kwargs["json"]["csl"] == ".show version":
file_name = "versionshowcommandresult.json"
else:
file_name = "adminthenquery.json"
with open(os.path.join(os.path.dirname(__file__), "input", file_name), "r") as response_file:
data = response_file.read()
return MockResponse(json.loads(data), 200, url)
elif url == "https://somecluster.kusto.windows.net/v1/rest/auth/metadata":
return MockResponse(
{
"AzureAD": {
"LoginEndpoint": "https://login.microsoftonline.com",
"LoginMfaRequired": False,
"KustoClientAppId": "db662dc1-0cfe-4e1c-a843-19a68e65be58",
"KustoClientRedirectUri": "https://microsoft/kustoclient",
"KustoServiceResourceId": "https://kusto.dev.kusto.windows.net",
"FirstPartyAuthorityUrl": "https://login.microsoftonline.com/f8cdef31-a31e-4b4a-93e4-5f571e91255a",
},
"dSTS": {
"CloudEndpointSuffix": "windows.net",
"DstsRealm": "realm://dsts.core.windows.net",
"DstsInstance": "prod-dsts.dsts.core.windows.net",
"KustoDnsHostName": "kusto.windows.net",
"ServiceName": "kusto",
},
},
200,
url,
)
return MockResponse(None, 404, url)
@pytest.fixture(
params=[
"user_password",
"application_key",
"application_token",
"device",
"user_token",
"managed_identity",
"token_provider",
"async_token_provider",
"az_cli",
"interactive_login",
]
)
def proxy_kcsb(request) -> Tuple[KustoConnectionStringBuilder, bool]:
cluster = KustoClientTestsMixin.HOST
user = "test2"
password = "Pa$$w0rd2"
authority_id = "13456"
uuid = "11111111-1111-1111-1111-111111111111"
key = "key of application"
token = "The app hardest token ever"
return {
"user_password": (KustoConnectionStringBuilder.with_aad_user_password_authentication(cluster, user, password, authority_id), True),
"application_key": (KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, uuid, key, "microsoft.com"), True),
"application_token": (KustoConnectionStringBuilder.with_aad_application_token_authentication(cluster, application_token=token), False),
"device": (KustoConnectionStringBuilder.with_aad_device_authentication(cluster), True),
"user_token": (KustoConnectionStringBuilder.with_aad_user_token_authentication(cluster, user_token=token), False),
"managed_identity": (KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster), False),
"token_provider": (KustoConnectionStringBuilder.with_token_provider(cluster, lambda x: x), False),
"async_token_provider": (KustoConnectionStringBuilder.with_async_token_provider(cluster, lambda x: x), False),
"az_cli": (KustoConnectionStringBuilder.with_az_cli_authentication(cluster), True),
"interactive_login": (KustoConnectionStringBuilder.with_interactive_login(cluster), True),
}[request.param]
DIGIT_WORDS = [str("Zero"), str("One"), str("Two"), str("Three"), str("Four"), str("Five"), str("Six"), str("Seven"), str("Eight"), str("Nine"), str("ten")]
SyncResponseSet = Union[KustoStreamingResponseDataSet, KustoResponseDataSet]
SyncResultTable = Union[KustoResultTable, KustoStreamingResultTable]
def get_response_first_primary_result(response: SyncResponseSet) -> SyncResultTable:
return next(x for x in response if x.table_kind == WellKnownDataSet.PrimaryResult)
def get_table_first_row(table: SyncResultTable) -> KustoResultRow:
return next(iter(table))
class KustoClientTestsMixin:
HOST = "https://somecluster.kusto.windows.net"
@staticmethod
def _assert_client_request_id(response_args: dict, value: Optional[str] = None) -> None:
header = response_args["headers"]["x-ms-client-request-id"]
if value:
assert header == value
return
[header_prefix, header_uuid] = header.split(";")
assert header_prefix == "KPC.execute"
uuid.UUID(header_uuid)
@staticmethod
def _assert_sanity_query_primary_results(results: Iterator[KustoResultRow]):
expected = {
"rownumber": None,
"rowguid": str(""),
"xdouble": None,
"xfloat": None,
"xbool": None,
"xint16": None,
"xint32": None,
"xint64": None,
"xuint8": None,
"xuint16": None,
"xuint32": None,
"xuint64": None,
"xdate": None,
"xsmalltext": str(""),
"xtext": str(""),
"xnumberAsText": str(""),
"xtime": None,
"xtextWithNulls": str(""),
"xdynamicWithNulls": str(""),
}
for row in results:
assert row["rownumber"] == expected["rownumber"]
assert row["rowguid"] == expected["rowguid"]
assert row["xdouble"] == expected["xdouble"]
assert row["xfloat"] == expected["xfloat"]
assert row["xbool"] == expected["xbool"]
assert row["xint16"] == expected["xint16"]
assert row["xint32"] == expected["xint32"]
assert row["xint64"] == expected["xint64"]
assert row["xuint8"] == expected["xuint8"]
assert row["xuint16"] == expected["xuint16"]
assert row["xuint32"] == expected["xuint32"]
assert row["xuint64"] == expected["xuint64"]
assert row["xdate"] == expected["xdate"]
assert row["xsmalltext"] == expected["xsmalltext"]
assert row["xtext"] == expected["xtext"]
assert row["xnumberAsText"] == expected["xnumberAsText"]
assert row["xtime"] == expected["xtime"]
assert row["xtextWithNulls"] == expected["xtextWithNulls"]
assert row["xdynamicWithNulls"] == expected["xdynamicWithNulls"]
assert isinstance(row["rownumber"], type(expected["rownumber"]))
assert isinstance(row["rowguid"], type(expected["rowguid"]))
assert isinstance(row["xdouble"], type(expected["xdouble"]))
assert isinstance(row["xfloat"], type(expected["xfloat"]))
assert isinstance(row["xbool"], type(expected["xbool"]))
assert isinstance(row["xint16"], type(expected["xint16"]))
assert isinstance(row["xint32"], type(expected["xint32"]))
assert isinstance(row["xint64"], type(expected["xint64"]))
assert isinstance(row["xuint8"], type(expected["xuint8"]))
assert isinstance(row["xuint16"], type(expected["xuint16"]))
assert isinstance(row["xuint32"], type(expected["xuint32"]))
assert isinstance(row["xuint64"], type(expected["xuint64"]))
assert isinstance(row["xdate"], type(expected["xdate"]))
assert isinstance(row["xsmalltext"], type(expected["xsmalltext"]))
assert isinstance(row["xtext"], type(expected["xtext"]))
assert isinstance(row["xnumberAsText"], type(expected["xnumberAsText"]))
assert isinstance(row["xtime"], type(expected["xtime"]))
assert isinstance(row["xtextWithNulls"], type(expected["xtextWithNulls"]))
assert isinstance(row["xdynamicWithNulls"], type(expected["xdynamicWithNulls"]))
expected["rownumber"] = 0 if expected["rownumber"] is None else expected["rownumber"] + 1
expected["rowguid"] = str("0000000{0}-0000-0000-0001-020304050607".format(expected["rownumber"]))
expected["xdouble"] = round(float(0) if expected["xdouble"] is None else expected["xdouble"] + 1.0001, 4)
expected["xfloat"] = round(float(0) if expected["xfloat"] is None else expected["xfloat"] + 1.01, 2)
expected["xbool"] = False if expected["xbool"] is None else not expected["xbool"]
expected["xint16"] = 0 if expected["xint16"] is None else expected["xint16"] + 1
expected["xint32"] = 0 if expected["xint32"] is None else expected["xint32"] + 1
expected["xint64"] = 0 if expected["xint64"] is None else expected["xint64"] + 1
expected["xuint8"] = 0 if expected["xuint8"] is None else expected["xuint8"] + 1
expected["xuint16"] = 0 if expected["xuint16"] is None else expected["xuint16"] + 1
expected["xuint32"] = 0 if expected["xuint32"] is None else expected["xuint32"] + 1
expected["xuint64"] = 0 if expected["xuint64"] is None else expected["xuint64"] + 1
expected["xdate"] = expected["xdate"] or datetime(2013, 1, 1, 1, 1, 1, 0, tzinfo=UTC)
expected["xdate"] = expected["xdate"].replace(year=expected["xdate"].year + 1)
expected["xsmalltext"] = DIGIT_WORDS[int(expected["xint16"])]
expected["xtext"] = DIGIT_WORDS[int(expected["xint16"])]
expected["xnumberAsText"] = str(expected["xint16"])
next_time = (
timedelta()
if expected["xtime"] is None
else (abs(expected["xtime"]) + timedelta(days=1, seconds=1, microseconds=1000)) * (-1) ** (expected["rownumber"] + 1)
)
# hacky tests - because time here is relative to previous row, after we pass a time where we have > 500 nanoseconds,
# another microseconds digit is needed
if expected["rownumber"] + 1 == 6:
next_time += timedelta(microseconds=1)
expected["xtime"] = next_time
if expected["xint16"] > 0:
expected["xdynamicWithNulls"] = {"rowId": expected["xint16"], "arr": [0, expected["xint16"]]}
@staticmethod
def _assert_sanity_query_response(response: SyncResponseSet):
KustoClientTestsMixin._assert_sanity_query_primary_results(get_response_first_primary_result(response))
@staticmethod
def _assert_sanity_control_command_response(response: SyncResponseSet):
assert len(response) == 1
primary_table = get_response_first_primary_result(response)
row_count = 0
for _ in primary_table:
row_count += 1
assert row_count == 1
result = primary_table[0]
assert result["BuildVersion"] == "1.0.6693.14577"
assert result["BuildTime"] == datetime(year=2018, month=4, day=29, hour=8, minute=5, second=54, tzinfo=UTC)
assert result["ServiceType"] == "Engine"
assert result["ProductVersion"] == "KustoMain_2018.04.29.5"
def _assert_sanity_data_frame_response(self, data_frame: "pandas.DataFrame"):
from pandas import DataFrame, Series
from pandas.testing import assert_frame_equal
assert len(data_frame.columns) == 19
expected_dict = {
"rownumber": Series([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="Int64"),
"rowguid": Series(
[
"",
"00000000-0000-0000-0001-020304050607",
"00000001-0000-0000-0001-020304050607",
"00000002-0000-0000-0001-020304050607",
"00000003-0000-0000-0001-020304050607",
"00000004-0000-0000-0001-020304050607",
"00000005-0000-0000-0001-020304050607",
"00000006-0000-0000-0001-020304050607",
"00000007-0000-0000-0001-020304050607",
"00000008-0000-0000-0001-020304050607",
"00000009-0000-0000-0001-020304050607",
],
dtype=object,
),
"xdouble": Series([None, 0.0, 1.0001, 2.0002, 3.0003, 4.0004, 5.0005, 6.0006, 7.0007, 8.0008, 9.0009], dtype="Float64"),
"xfloat": Series([None, 0.0, 1.01, 2.02, 3.03, 4.04, 5.05, 6.06, 7.07, 8.08, 9.09], dtype="Float64"),
"xbool": Series([None, False, True, False, True, False, True, False, True, False, True], dtype=bool),
"xint16": Series([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="Int64"),
"xint32": Series([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="Int64"),
"xint64": Series([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="Int64"),
"xuint8": Series([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="Int64"),
"xuint16": Series([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="Int64"),
"xuint32": Series([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="Int64"),
"xuint64": Series([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="Int64"),
"xdate": Series(
[
pandas.to_datetime(None),
pandas.to_datetime("2014-01-01T01:01:01.0000000Z"),
pandas.to_datetime("2015-01-01T01:01:01.0000001Z"),
pandas.to_datetime("2016-01-01T01:01:01.0000002Z"),
pandas.to_datetime("2017-01-01T01:01:01.0000003Z"),
pandas.to_datetime("2018-01-01T01:01:01.0000004Z"),
pandas.to_datetime("2019-01-01T01:01:01.0000005Z"),
pandas.to_datetime("2020-01-01T01:01:01.0000006Z"),
pandas.to_datetime("2021-01-01T01:01:01.0000007Z"),
pandas.to_datetime("2022-01-01T01:01:01.0000008Z"),
pandas.to_datetime("2023-01-01T01:01:01.0000009Z"),
]
),
"xsmalltext": Series(["", "Zero", "One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine"], dtype=object),
"xtext": Series(["", "Zero", "One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine"], dtype=object),
"xnumberAsText": Series(["", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"], dtype=object),
"xtime": Series(
[
"NaT",
0,
"1 days 00:00:01.0010001",
"-2 days 00:00:02.0020002",
"3 days 00:00:03.0030003",
"-4 days 00:00:04.0040004",
"5 days 00:00:05.0050005",
"-6 days 00:00:06.0060006",
"7 days 00:00:07.0070007",
"-8 days 00:00:08.0080008",
"9 days 00:00:09.0090009",
],
dtype="timedelta64[ns]",
),
"xtextWithNulls": Series(["", "", "", "", "", "", "", "", "", "", ""], dtype=object),
"xdynamicWithNulls": Series(
[
str(""),
str(""),
{"rowId": 1, "arr": [0, 1]},
{"rowId": 2, "arr": [0, 2]},
{"rowId": 3, "arr": [0, 3]},
{"rowId": 4, "arr": [0, 4]},
{"rowId": 5, "arr": [0, 5]},
{"rowId": 6, "arr": [0, 6]},
{"rowId": 7, "arr": [0, 7]},
{"rowId": 8, "arr": [0, 8]},
{"rowId": 9, "arr": [0, 9]},
],
dtype=object,
),
}
columns = [
"rownumber",
"rowguid",
"xdouble",
"xfloat",
"xbool",
"xint16",
"xint32",
"xint64",
"xuint8",
"xuint16",
"xuint32",
"xuint64",
"xdate",
"xsmalltext",
"xtext",
"xnumberAsText",
"xtime",
"xtextWithNulls",
"xdynamicWithNulls",
]
expected_data_frame = DataFrame(expected_dict, columns=columns, copy=True)
assert_frame_equal(data_frame, expected_data_frame)
@staticmethod
def _assert_partial_results_response(response: SyncResponseSet):
results = list(get_response_first_primary_result(response))
assert len(results) == 5
assert results[0]["x"] == 1
if type(response) == KustoStreamingResponseDataSet:
_ = [t for t in response] # Read rest of tables
assert response.errors_count == 1
assert "E_QUERY_RESULT_SET_TOO_LARGE" in response.get_exceptions()[0]
assert len(response) == 3
@staticmethod
def _assert_admin_then_query_response(response: SyncResponseSet):
assert response.errors_count == 0
assert len(response) == 4
results = list(get_response_first_primary_result(response))
assert len(results) == 2
assert response[0].table_kind == WellKnownDataSet.PrimaryResult
assert response[1].table_kind == WellKnownDataSet.QueryProperties
assert response[2].table_kind == WellKnownDataSet.QueryCompletionInformation
assert response[3].table_kind == WellKnownDataSet.TableOfContents
@staticmethod
def _assert_dynamic_response(row: KustoResultRow):
assert isinstance(row[0], int)
assert row[0] == 123
assert isinstance(row[1], str)
assert row[1] == "123"
assert isinstance(row[2], str)
assert row[2] == "test bad json"
assert row[3] is None
assert isinstance(row[4], str)
assert row[4] == '{"rowId":2,"arr":[0,2]}'
assert isinstance(row[5], dict)
assert row[5] == {"rowId": 2, "arr": [0, 2]}
azure-kusto-python-3.0.1/azure-kusto-data/tests/sample.py 0000664 0000000 0000000 00000015730 14172227630 0023532 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
"""A simple example how to use KustoClient."""
from datetime import timedelta
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties
from azure.kusto.data.exceptions import KustoServiceError
from azure.kusto.data.helpers import dataframe_from_result_table
######################################################
## AUTH ##
######################################################
# Note that the 'help' cluster only allows interactive
# access by AAD users (and *not* AAD applications)
cluster = "https://help.kusto.windows.net"
# In case you want to authenticate with AAD application.
client_id = ""
client_secret = ""
# read more at https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id
authority_id = ""
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, client_id, client_secret, authority_id)
# In case you want to authenticate with AAD application certificate.
filename = "path to a PEM certificate"
with open(filename, "r") as pem_file:
PEM = pem_file.read()
thumbprint = "certificate's thumbprint"
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(cluster, client_id, PEM, thumbprint, authority_id)
# In case you want to authenticate with AAD application certificate Subject Name & Issuer
filename = "path to a PEM certificate"
with open(filename, "r") as pem_file:
PEM = pem_file.read()
filename = "path to a public certificate"
with open(filename, "r") as cert_file:
public_certificate = cert_file.read()
thumbprint = "certificate's thumbprint"
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_sni_authentication(cluster, client_id, PEM, public_certificate, thumbprint, authority_id)
# In case you want to authenticate with a System Assigned Managed Service Identity (MSI)
kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster)
# In case you want to authenticate with a User Assigned Managed Service Identity (MSI)
user_assigned_client_id = "the AAD identity client id"
kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster, client_id=user_assigned_client_id)
# In case you want to authenticate with Azure CLI.
# Users are required to be in a logged in state in az-cli, for this authentication method to succeed. Run `az login` to login to azure cli.
kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(cluster)
# In case you want to authenticate with AAD username and password
username = ""
password = ""
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(cluster, username, password, authority_id)
# In case you want to authenticate with AAD device code.
# Please note that if you choose this option, you'll need to authenticate for every new instance that is initialized.
# It is highly recommended to create one instance and use it for all of your queries.
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster)
# The authentication method will be taken from the chosen KustoConnectionStringBuilder.
client = KustoClient(kcsb)
######################################################
## QUERY ##
######################################################
# once authenticated, usage is as following
db = "Samples"
query = "StormEvents | take 10"
response = client.execute(db, query)
# iterating over rows is possible
for row in response.primary_results[0]:
# printing specific columns by index
print("value at 0 {}".format(row[0]))
print("\n")
# printing specific columns by name
print("EventType:{}".format(row["EventType"]))
# tables are serializeable, so:
with open("results.json", "w+") as f:
f.write(str(response.primary_results[0]))
# we also support dataframes:
dataframe = dataframe_from_result_table(response.primary_results[0])
print(dataframe)
# Streaming Query - rather than reading everything ahead, iterate through results as they come
multiple_tables = 'StormEvents | where EventType == "Heavy Rain" | take 10; StormEvents | where EventType == "Tornado" | take 10'
results = client.execute_streaming_query("DB", multiple_tables)
tables_iter = results.iter_primary_results()
first_table = next(tables_iter)
# Will block until each row arrives
for row in first_table:
# printing specific columns by index
print("value at 0 {}".format(row[0]))
print("\n")
# printing specific columns by name
print("EventType:{}".format(row["EventType"]))
# next(tables_iter) - throws, we can't read the next table until we exhausted this one
# Read next table
second_table = next(tables_iter)
print(next(second_table))
# You can always access the table's properties, even after it's exhausted
print(first_table.columns, first_table.table_kind, first_table.table_name)
print(second_table.columns, second_table.table_kind, second_table.table_name)
# Will skip forward, but the previous table will be exhausted
results.set_skip_incomplete_tables(True)
next(results, None) # Tables are finished
# When we finish all the tables we get None
print(results.tables) # Access all tables, not just the primary results
##################
### EXCEPTIONS ###
##################
# Query is too big to be executed
query = "StormEvents"
try:
response = client.execute(db, query)
except KustoServiceError as error:
print("2. Error:", error)
print("2. Is semantic error:", error.is_semantic_error())
print("2. Has partial results:", error.has_partial_results())
print("2. Result size:", len(error.get_partial_results()))
properties = ClientRequestProperties()
properties.set_option(properties.results_defer_partial_query_failures_option_name, True)
properties.set_option(properties.request_timeout_option_name, timedelta(seconds=8 * 60))
response = client.execute(db, query, properties=properties)
print("3. Response error count: ", response.errors_count)
print("3. Exceptions:", response.get_exceptions())
print("3. Result size:", len(response.primary_results))
# Query has semantic error
query = "StormEvents | where foo = bar"
try:
response = client.execute(db, query)
except KustoServiceError as error:
print("4. Error:", error)
print("4. Is semantic error:", error.is_semantic_error())
print("4. Has partial results:", error.has_partial_results())
client = KustoClient("https://kustolab.kusto.windows.net")
response = client.execute("ML", ".show version")
query = """
let max_t = datetime(2016-09-03);
service_traffic
| make-series num=count() on TimeStamp in range(max_t-5d, max_t, 1h) by OsVer
"""
response = client.execute_query("ML", query).primary_results[0]
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_client_request_properties.py 0000664 0000000 0000000 00000002270 14172227630 0030605 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import unittest
from datetime import timedelta
from azure.kusto.data import ClientRequestProperties
class ClientRequestPropertiesTests(unittest.TestCase):
"""ClientRequestProperties Tests"""
def test_properties(self):
"""positive tests"""
defer = False
timeout = timedelta(seconds=10)
crp = ClientRequestProperties()
crp.set_option(ClientRequestProperties.results_defer_partial_query_failures_option_name, defer)
crp.set_option(ClientRequestProperties.request_timeout_option_name, timeout)
result = crp.to_json()
assert '"{0}": false'.format(crp.results_defer_partial_query_failures_option_name) in result
assert '"{0}": "0:00:10"'.format(ClientRequestProperties.request_timeout_option_name) in result
assert crp.client_request_id is None
assert crp.application is None
assert crp.user is None
crp.client_request_id = "CRID"
assert crp.client_request_id == "CRID"
crp.application = "myApp"
assert crp.application == "myApp"
crp.user = "myUser"
assert crp.user == "myUser"
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_converter.py 0000664 0000000 0000000 00000005055 14172227630 0025316 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import unittest
from datetime import timedelta
from azure.kusto.data._converters import to_datetime, to_timedelta
class ConverterTests(unittest.TestCase):
"""These are unit tests that should test custom converters used in."""
def test_to_timestamp(self):
"""Happy path to test converter from TimeSpan to timedelta."""
# Test hours, minutes and seconds
assert to_timedelta("00:00:00") == timedelta(seconds=0)
assert to_timedelta("00:00:03") == timedelta(seconds=3)
assert to_timedelta("00:04:03") == timedelta(minutes=4, seconds=3)
assert to_timedelta("02:04:03") == timedelta(hours=2, minutes=4, seconds=3)
# Test milliseconds
assert to_timedelta("00:00:00.099") == timedelta(milliseconds=99)
assert to_timedelta("02:04:03.0123") == timedelta(hours=2, minutes=4, seconds=3, microseconds=12300)
# Test days
assert to_timedelta("01.00:00:00") == timedelta(days=1)
assert to_timedelta("02.04:05:07") == timedelta(days=2, hours=4, minutes=5, seconds=7)
# Test negative
assert to_timedelta("-01.00:00:00") == -timedelta(days=1)
assert to_timedelta("-02.04:05:07") == -timedelta(days=2, hours=4, minutes=5, seconds=7)
# Test all together
assert to_timedelta("00.00:00:00.000") == timedelta(seconds=0)
assert to_timedelta("02.04:05:07.789") == timedelta(days=2, hours=4, minutes=5, seconds=7, milliseconds=789)
assert to_timedelta("03.00:00:00.111") == timedelta(days=3, milliseconds=111)
# Test from Ticks
assert to_timedelta(-80080008) == timedelta(microseconds=-8008001)
assert to_timedelta(10010001) == timedelta(microseconds=1001000)
def test_to_timestamp_fail(self):
"""
Sad path to test TimeSpan to timedelta converter
"""
self.assertRaises(ValueError, to_timedelta, "")
self.assertRaises(ValueError, to_timedelta, "foo")
self.assertRaises(ValueError, to_timedelta, "00")
self.assertRaises(ValueError, to_timedelta, "00:00")
self.assertRaises(ValueError, to_timedelta, "03.00:00:00.")
self.assertRaises(ValueError, to_timedelta, "03.00:00:00.111a")
def test_to_datetime(self):
"""Tests datetime read by KustoResultIter"""
assert to_datetime("2016-06-07T16:00:00Z") is not None
def test_to_datetime_fail(self):
"""Tests that invalid strings fails to convert to datetime"""
self.assertRaises(ValueError, to_datetime, "invalid")
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_functional.py 0000664 0000000 0000000 00000014401 14172227630 0025444 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import json
import unittest
from datetime import datetime, timedelta
from azure.kusto.data.response import KustoResponseDataSetV2
from dateutil.tz.tz import tzutc
# Sample response against all tests should be run
RESPONSE_TEXT = """
[{
"FrameType": "DataSetHeader",
"IsProgressive": false,
"Version": "v2.0"
},
{
"FrameType": "DataTable",
"TableId": 0,
"TableName": "@ExtendedProperties",
"TableKind": "QueryProperties",
"Columns": [{
"ColumnName": "TableId",
"ColumnType": "int"
},
{
"ColumnName": "Key",
"ColumnType": "string"
},
{
"ColumnName": "Value",
"ColumnType": "dynamic"
}],
"Rows": [[1,
"Visualization",
"{\\"Visualization\\":null,\\"Title\\":null,\\"XColumn\\":null,\\"Series\\":null,\\"YColumns\\":null,\\"XTitle\\":null,\\"YTitle\\":null,\\"XAxis\\":null,\\"YAxis\\":null,\\"Legend\\":null,\\"YSplit\\":null,\\"Accumulate\\":false,\\"IsQuerySorted\\":false,\\"Kind\\":null}"]]
},
{
"FrameType": "DataTable",
"TableId": 1,
"TableName": "temp",
"TableKind": "PrimaryResult",
"Columns": [{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "Name",
"ColumnType": "string"
},
{
"ColumnName": "Altitude",
"ColumnType": "long"
},
{
"ColumnName": "Temperature",
"ColumnType": "real"
},
{
"ColumnName": "IsFlying",
"ColumnType": "bool"
},
{
"ColumnName": "TimeFlying",
"ColumnType": "timespan"
}],
"Rows": [["2016-06-06T15:35:00Z",
"foo",
101,
3.14,
false,
3493235670000],
["2016-06-07T16:00:00Z",
"bar",
555,
2.71,
true,
0],
[null,
"",
null,
null,
null,
null]]
},
{
"FrameType": "DataTable",
"TableId": 2,
"TableName": "QueryCompletionInformation",
"TableKind": "QueryCompletionInformation",
"Columns": [{
"ColumnName": "Timestamp",
"ColumnType": "datetime"
},
{
"ColumnName": "ClientRequestId",
"ColumnType": "string"
},
{
"ColumnName": "ActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "ParentActivityId",
"ColumnType": "guid"
},
{
"ColumnName": "Level",
"ColumnType": "int"
},
{
"ColumnName": "LevelName",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"ColumnType": "int"
},
{
"ColumnName": "StatusCodeName",
"ColumnType": "string"
},
{
"ColumnName": "EventType",
"ColumnType": "int"
},
{
"ColumnName": "EventTypeName",
"ColumnType": "string"
},
{
"ColumnName": "Payload",
"ColumnType": "string"
}],
"Rows": [["2018-05-01T09:32:38.916566Z",
"unspecified;e8e72755-786b-4bdc-835d-ea49d63d09fd",
"5935a050-e466-48a0-991d-0ec26bd61c7e",
"8182b177-7a80-4158-aca8-ff4fd8e7d3f8",
"6f3c1072-2739-461c-8aa7-3cfc8ff528a8",
4,
"Info",
0,
"S_OK (0)",
4,
"QueryInfo",
"{\\"Count\\":1,\\"Text\\":\\"Querycompletedsuccessfully\\"}"],
["2018-05-01T09:32:38.916566Z",
"unspecified;e8e72755-786b-4bdc-835d-ea49d63d09fd",
"5935a050-e466-48a0-991d-0ec26bd61c7e",
"8182b177-7a80-4158-aca8-ff4fd8e7d3f8",
"6f3c1072-2739-461c-8aa7-3cfc8ff528a8",
6,
"Stats",
0,
"S_OK (0)",
0,
"QueryResourceConsumption",
"{\\"ExecutionTime\\":0.0156222,\\"resource_usage\\":{\\"cache\\":{\\"memory\\":{\\"hits\\":13,\\"misses\\":0,\\"total\\":13},\\"disk\\":{\\"hits\\":0,\\"misses\\":0,\\"total\\":0}},\\"cpu\\":{\\"user\\":\\"00: 00: 00\\",\\"kernel\\":\\"00: 00: 00\\",\\"totalcpu\\":\\"00: 00: 00\\"},\\"memory\\":{\\"peak_per_node\\":16777312}},\\"dataset_statistics\\":[{\\"table_row_count\\":3,\\"table_size\\":191}]}"]]
},
{
"FrameType": "DataSetCompletion",
"HasErrors": false,
"Cancelled": false
}]
"""
class FunctionalTests(unittest.TestCase):
"""E2E tests, from part where response is received from Kusto. This does not include access to actual Kusto cluster."""
def test_valid_response(self):
"""Tests on happy path, validating response and iterations over it."""
response = KustoResponseDataSetV2(json.loads(RESPONSE_TEXT))
# Test that basic iteration works
assert len(response) == 3
assert len(list(response.primary_results[0])) == 3
table = list(response.tables[0])
assert 1 == len(table)
expected_table = [
[datetime(2016, 6, 6, 15, 35, tzinfo=tzutc()), "foo", 101, 3.14, False, timedelta(days=4, hours=1, minutes=2, seconds=3, milliseconds=567)],
[datetime(2016, 6, 7, 16, tzinfo=tzutc()), "bar", 555, 2.71, True, timedelta()],
[None, str(""), None, None, None, None],
]
columns = ["Timestamp", "Name", "Altitude", "Temperature", "IsFlying", "TimeFlying"]
# Test access by index and by column name
primary_table = response.primary_results[0]
for row in primary_table:
# Test all types
for i, expected_type in enumerate([datetime, str, int, float, bool, timedelta]):
assert row[i] == row[columns[i]]
assert row[i] is None or isinstance(row[i], expected_type)
for row_index, row in enumerate(primary_table):
expected_row = expected_table[row_index]
for col_index, value in enumerate(row):
assert value == expected_row[col_index]
def test_invalid_table(self):
"""Tests calling of table with index that doesn't exists."""
response = KustoResponseDataSetV2(json.loads(RESPONSE_TEXT))
self.assertRaises(IndexError, response.__getitem__, 7)
self.assertRaises(IndexError, response.__getitem__, -6)
def test_column_dont_exist(self):
"""Tests accessing column that doesn't exists."""
response = KustoResponseDataSetV2(json.loads(RESPONSE_TEXT))
row = response.primary_results[0][0]
self.assertRaises(IndexError, row.__getitem__, 10)
self.assertRaises(LookupError, row.__getitem__, "NonexistentColumn")
def test_iterating_after_end(self):
"""Tests StopIteration is raised when the response ends."""
response = KustoResponseDataSetV2(json.loads(RESPONSE_TEXT))
assert sum(1 for _ in response.primary_results[0]) == 3
def test_row_equality(self):
"""Tests the rows are idempotent."""
response = KustoResponseDataSetV2(json.loads(RESPONSE_TEXT))
table = response.primary_results[0]
for row_index, row in enumerate(table):
assert table[row_index] == row
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_helpers.py 0000664 0000000 0000000 00000006414 14172227630 0024751 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import json
import os
import pytest
from azure.kusto.data.helpers import dataframe_from_result_table
from azure.kusto.data.response import KustoResponseDataSetV2
PANDAS = False
try:
import pandas
import numpy
PANDAS = True
except:
pass
class TestDataFrameFromResultsTable:
"""Tests the dataframe_from_result_table helper function"""
@pytest.mark.skipif(not PANDAS, reason="requires pandas")
def test_dataframe_from_result_table(self):
"""Test conversion of KustoResultTable to pandas.DataFrame, including fixes for certain column types"""
with open(os.path.join(os.path.dirname(__file__), "input", "dataframe.json"), "r") as response_file:
data = response_file.read()
response = KustoResponseDataSetV2(json.loads(data))
df = dataframe_from_result_table(response.primary_results[0])
assert df.iloc[0].RecordName == "now"
assert type(df.iloc[0].RecordTime) is pandas._libs.tslibs.timestamps.Timestamp
assert all(getattr(df.iloc[0].RecordTime, k) == v for k, v in {"year": 2021, "month": 12, "day": 22, "hour": 11, "minute": 43, "second": 00}.items())
assert type(df.iloc[0].RecordBool) is numpy.bool_
assert df.iloc[0].RecordBool == True
assert type(df.iloc[0].RecordInt) is numpy.int64
assert df.iloc[0].RecordInt == 5678
assert type(df.iloc[0].RecordReal) is numpy.float64
assert df.iloc[0].RecordReal == 3.14159
# Kusto datetime(0000-01-01T00:00:00Z), which Pandas can't represent.
assert df.iloc[1].RecordName == "earliest datetime"
assert type(df.iloc[1].RecordTime) is pandas._libs.tslibs.nattype.NaTType
assert pandas.isnull(df.iloc[1].RecordReal)
# Kusto datetime(9999-12-31T23:59:59Z), which Pandas can't represent.
assert df.iloc[2].RecordName == "latest datetime"
assert type(df.iloc[2].RecordTime) is pandas._libs.tslibs.nattype.NaTType
assert type(df.iloc[2].RecordReal) is numpy.float64
assert df.iloc[2].RecordReal == numpy.inf
# Pandas earliest datetime
assert df.iloc[3].RecordName == "earliest pandas datetime"
assert type(df.iloc[3].RecordTime) is pandas._libs.tslibs.timestamps.Timestamp
assert type(df.iloc[3].RecordReal) is numpy.float64
assert df.iloc[3].RecordReal == -numpy.inf
# Pandas latest datetime
assert df.iloc[4].RecordName == "latest pandas datetime"
assert type(df.iloc[4].RecordTime) is pandas._libs.tslibs.timestamps.Timestamp
# Kusto 600000000 ticks timedelta
assert df.iloc[5].RecordName == "timedelta ticks"
assert type(df.iloc[5].RecordTime) is pandas._libs.tslibs.timestamps.Timestamp
assert type(df.iloc[5].RecordOffset) is pandas._libs.tslibs.timestamps.Timedelta
assert df.iloc[5].RecordOffset == pandas.to_timedelta("00:01:00")
# Kusto timedelta(1.01:01:01.0) ==
assert df.iloc[6].RecordName == "timedelta string"
assert type(df.iloc[6].RecordTime) is pandas._libs.tslibs.timestamps.Timestamp
assert type(df.iloc[6].RecordOffset) is pandas._libs.tslibs.timestamps.Timedelta
assert df.iloc[6].RecordOffset == pandas.to_timedelta("1 days 01:01:01")
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_kusto_client.py 0000664 0000000 0000000 00000016273 14172227630 0026016 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import sys
import pytest
from mock import patch
from azure.kusto.data import KustoClient, ClientRequestProperties
from azure.kusto.data._cloud_settings import CloudSettings
from azure.kusto.data.exceptions import KustoMultiApiError
from azure.kusto.data.helpers import dataframe_from_result_table
from azure.kusto.data.response import KustoStreamingResponseDataSet
from tests.kusto_client_common import KustoClientTestsMixin, mocked_requests_post, get_response_first_primary_result, get_table_first_row, proxy_kcsb
PANDAS = False
try:
import pandas
PANDAS = True
except:
pass
@pytest.fixture(params=[KustoClient.execute_query, KustoClient.execute_streaming_query])
def method(request):
return request.param
class TestKustoClient(KustoClientTestsMixin):
"""Tests class for KustoClient API"""
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_sanity_query(self, mock_post, method):
"""Test query V2."""
client = KustoClient(self.HOST)
response = method.__call__(client, "PythonTest", "Deft")
self._assert_sanity_query_response(response)
self._assert_client_request_id(mock_post.call_args.kwargs)
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_sanity_control_command(self, mock_post):
"""Tests contol command."""
client = KustoClient(self.HOST)
response = client.execute_mgmt("NetDefaultDB", ".show version")
self._assert_sanity_control_command_response(response)
@pytest.mark.skipif(not PANDAS, reason="requires pandas")
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_sanity_data_frame(self, mock_post, method):
"""Tests KustoResponse to pandas.DataFrame."""
client = KustoClient(self.HOST)
response = method.__call__(client, "PythonTest", "Deft")
data_frame = dataframe_from_result_table(get_response_first_primary_result(response))
self._assert_sanity_data_frame_response(data_frame)
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_partial_results(self, mock_post, method):
"""Tests partial results."""
client = KustoClient(self.HOST)
query = """set truncationmaxrecords = 5;
range x from 1 to 10 step 1"""
properties = ClientRequestProperties()
properties.set_option(ClientRequestProperties.results_defer_partial_query_failures_option_name, False)
with pytest.raises(KustoMultiApiError) as e:
response = method.__call__(client, "PythonTest", query, properties=properties)
if type(response) == KustoStreamingResponseDataSet:
results = list(get_response_first_primary_result(response))
errors = e.value.get_api_errors()
assert len(errors) == 1
assert errors[0].code == "LimitsExceeded"
properties.set_option(ClientRequestProperties.results_defer_partial_query_failures_option_name, True)
response = method.__call__(client, "PythonTest", query, properties=properties)
self._assert_partial_results_response(response)
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_admin_then_query(self, mock_post):
"""Tests admin then query."""
client = KustoClient(self.HOST)
query = ".show tables | project DatabaseName, TableName"
response = client.execute_mgmt("PythonTest", query)
self._assert_admin_then_query_response(response)
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_dynamic(self, mock_post, method):
"""Tests dynamic responses."""
client = KustoClient(self.HOST)
query = """print dynamic(123), dynamic("123"), dynamic("test bad json"),"""
""" dynamic(null), dynamic('{"rowId":2,"arr":[0,2]}'), dynamic({"rowId":2,"arr":[0,2]})"""
row = get_table_first_row(get_response_first_primary_result(method.__call__(client, "PythonTest", query)))
self._assert_dynamic_response(row)
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_empty_result(self, mock_post, method):
"""Tests dynamic responses."""
client = KustoClient(self.HOST)
query = """print 'a' | take 0"""
response = method.__call__(client, "PythonTest", query)
assert get_response_first_primary_result(response)
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_null_values_in_data(self, mock_post, method):
"""Tests response with null values in non nullable column types"""
client = KustoClient(self.HOST)
query = "PrimaryResultName"
response = method.__call__(client, "PythonTest", query)
assert response is not None
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_unidentifiable_os(self, mock_post, method):
"""Tests unidentifiable OS doesn't fail when composing its socket options"""
with patch.object(sys, "platform", "win3.1"):
client = KustoClient("https://somecluster.kusto.windows.net")
query = """print dynamic(123)"""
row = get_table_first_row(get_response_first_primary_result(method.__call__(client, "PythonTest", query)))
assert isinstance(row[0], int)
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_identifiable_os(self, mock_post, method):
"""Tests identifiable OS doesn't fail when composing its socket options"""
with patch.object(sys, "platform", "win32"):
client = KustoClient("https://somecluster.kusto.windows.net")
query = """print dynamic(123)"""
row = get_table_first_row(get_response_first_primary_result(method.__call__(client, "PythonTest", query)))
assert isinstance(row[0], int)
@patch("requests.Session.post", side_effect=mocked_requests_post)
def test_custom_request_id(self, mock_post, method):
"""Test query V2."""
client = KustoClient(self.HOST)
properties = ClientRequestProperties()
request_id = "test_request_id"
properties.client_request_id = request_id
response = method.__call__(client, "PythonTest", "Deft", properties=properties)
self._assert_sanity_query_response(response)
self._assert_client_request_id(mock_post.call_args.kwargs, value=request_id)
@patch("requests.get", side_effect=mocked_requests_post)
def test_proxy_token_providers(self, mock_get, proxy_kcsb):
"""Test query V2."""
proxy = "https://my_proxy.sample"
kcsb, auth_supports_proxy = proxy_kcsb
client = KustoClient(kcsb)
client.set_proxy(proxy)
assert client._proxy_url == proxy
expected_dict = {"http": proxy, "https": proxy}
if not auth_supports_proxy:
return
assert client._aad_helper.token_provider._proxy_dict == expected_dict
assert client._session.proxies == expected_dict
CloudSettings._cloud_cache.clear()
client._aad_helper.token_provider._init_resources()
mock_get.assert_called_with("https://somecluster.kusto.windows.net/v1/rest/auth/metadata", proxies=expected_dict)
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_kusto_connection_string_builder.py 0000664 0000000 0000000 00000034134 14172227630 0031767 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import unittest
from uuid import uuid4
from azure.kusto.data import KustoConnectionStringBuilder
class KustoConnectionStringBuilderTests(unittest.TestCase):
"""Tests class for KustoConnectionStringBuilder."""
PASSWORDS_REPLACEMENT = "****"
def test_no_credentials(self):
"""Checks kcsb that is created with no credentials"""
kcsbs = [
KustoConnectionStringBuilder("localhost"),
KustoConnectionStringBuilder("data Source=localhost"),
KustoConnectionStringBuilder("Addr=localhost"),
KustoConnectionStringBuilder("Addr = localhost"),
]
for kcsb in kcsbs:
assert kcsb.data_source == "localhost"
assert not kcsb.aad_federated_security
assert kcsb.aad_user_id is None
assert kcsb.password is None
assert kcsb.application_client_id is None
assert kcsb.application_key is None
assert kcsb.authority_id == "common"
assert repr(kcsb) == "Data Source=localhost;Authority Id=common"
assert str(kcsb) == "Data Source=localhost;Authority Id=common"
def test_aad_app(self):
"""Checks kcsb that is created with AAD application credentials."""
uuid = str(uuid4())
key = "key of application"
kcsbs = [
KustoConnectionStringBuilder(
"localhost;Application client Id={0};application Key={1};Authority Id={2} ; aad federated security = {3}".format(
uuid, key, "microsoft.com", True
)
),
KustoConnectionStringBuilder(
"Data Source=localhost ; Application Client Id={0}; Appkey ={1};Authority Id= {2} ; aad federated security = {3}".format(
uuid, key, "microsoft.com", True
)
),
KustoConnectionStringBuilder(
" Addr = localhost ; AppClientId = {0} ; AppKey ={1}; Authority Id={2} ; aad federated security = {3}".format(uuid, key, "microsoft.com", True)
),
KustoConnectionStringBuilder(
"Network Address = localhost; AppClientId = {0} ; AppKey ={1};AuthorityId={2} ; aad federated security = {3}".format(
uuid, key, "microsoft.com", True
)
),
KustoConnectionStringBuilder.with_aad_application_key_authentication("localhost", uuid, key, "microsoft.com"),
]
try:
KustoConnectionStringBuilder.with_aad_application_key_authentication("localhost", uuid, key, None)
except Exception as e:
# make sure error is raised when authority_id i none
assert isinstance(e, ValueError)
kcsb1 = KustoConnectionStringBuilder("server=localhost")
kcsb1[KustoConnectionStringBuilder.ValidKeywords.application_client_id] = uuid
kcsb1[KustoConnectionStringBuilder.ValidKeywords.application_key] = key
kcsb1[KustoConnectionStringBuilder.ValidKeywords.authority_id] = "microsoft.com"
kcsb1[KustoConnectionStringBuilder.ValidKeywords.aad_federated_security] = True
kcsbs.append(kcsb1)
kcsb2 = KustoConnectionStringBuilder("Server=localhost")
kcsb2["AppclientId"] = uuid
kcsb2["Application key"] = key
kcsb2["Authority Id"] = "microsoft.com"
kcsb2["aad federated security"] = True
kcsbs.append(kcsb2)
for kcsb in kcsbs:
assert kcsb.data_source == "localhost"
assert kcsb.aad_federated_security
assert kcsb.aad_user_id is None
assert kcsb.password is None
assert kcsb.application_client_id == uuid
assert kcsb.application_key == key
assert kcsb.authority_id == "microsoft.com"
assert repr(kcsb) == "Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}".format(
uuid, key, "microsoft.com"
)
assert str(kcsb) == "Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}".format(
uuid, self.PASSWORDS_REPLACEMENT, "microsoft.com"
)
def test_aad_user(self):
"""Checks kcsb that is created with AAD user credentials."""
user = "test"
password = "Pa$$w0rd"
kcsbs = [
KustoConnectionStringBuilder("localhost;AAD User ID={0};password={1} ;AAD Federated Security=True ".format(user, password)),
KustoConnectionStringBuilder("Data Source=localhost ; AaD User ID={0}; Password ={1} ;AAD Federated Security=True".format(user, password)),
KustoConnectionStringBuilder(" Addr = localhost ; AAD User ID = {0} ; Pwd ={1} ;AAD Federated Security=True".format(user, password)),
KustoConnectionStringBuilder("Network Address = localhost; AAD User iD = {0} ; Pwd = {1} ;AAD Federated Security= True ".format(user, password)),
KustoConnectionStringBuilder.with_aad_user_password_authentication("localhost", user, password),
]
kcsb1 = KustoConnectionStringBuilder("Server=localhost")
kcsb1[KustoConnectionStringBuilder.ValidKeywords.aad_user_id] = user
kcsb1[KustoConnectionStringBuilder.ValidKeywords.password] = password
kcsb1[KustoConnectionStringBuilder.ValidKeywords.aad_federated_security] = True
kcsbs.append(kcsb1)
kcsb2 = KustoConnectionStringBuilder("server=localhost")
kcsb2["AAD User ID"] = user
kcsb2["Password"] = password
kcsb2["aad federated security"] = True
kcsbs.append(kcsb2)
for kcsb in kcsbs:
assert kcsb.data_source == "localhost"
assert kcsb.aad_federated_security
assert kcsb.aad_user_id == user
assert kcsb.password == password
assert kcsb.application_client_id is None
assert kcsb.application_key is None
assert kcsb.authority_id == "common"
assert repr(kcsb) == "Data Source=localhost;AAD Federated Security=True;AAD User ID={0};Password={1};Authority Id=common".format(user, password)
assert str(kcsb) == "Data Source=localhost;AAD Federated Security=True;AAD User ID={0};Password={1};Authority Id=common".format(
user, self.PASSWORDS_REPLACEMENT
)
def test_aad_user_with_authority(self):
"""Checks kcsb that is created with AAD user credentials."""
user = "test2"
password = "Pa$$w0rd2"
authority_id = "13456"
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication("localhost", user, password, authority_id)
assert kcsb.data_source == "localhost"
assert kcsb.aad_federated_security
assert kcsb.aad_user_id == user
assert kcsb.password == password
assert kcsb.application_client_id is None
assert kcsb.application_key is None
assert kcsb.authority_id == authority_id
assert repr(kcsb) == "Data Source=localhost;AAD Federated Security=True;AAD User ID={0};Password={1};Authority Id=13456".format(user, password)
assert str(kcsb) == "Data Source=localhost;AAD Federated Security=True;AAD User ID={0};Password={1};Authority Id=13456".format(
user, self.PASSWORDS_REPLACEMENT
)
def test_aad_device_login(self):
"""Checks kcsb that is created with AAD device login."""
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication("localhost")
assert kcsb.data_source == "localhost"
assert kcsb.aad_federated_security
assert kcsb.aad_user_id is None
assert kcsb.password is None
assert kcsb.application_client_id is None
assert kcsb.application_key is None
assert kcsb.authority_id == "common"
assert repr(kcsb) == "Data Source=localhost;AAD Federated Security=True;Authority Id=common"
assert str(kcsb) == "Data Source=localhost;AAD Federated Security=True;Authority Id=common"
def test_aad_app_token(self):
"""Checks kcsb that is created with AAD user token."""
token = "The app hardest token ever"
kcsb = KustoConnectionStringBuilder.with_aad_application_token_authentication("localhost", application_token=token)
assert kcsb.data_source == "localhost"
assert kcsb.application_token == token
assert kcsb.aad_federated_security
assert kcsb.aad_user_id is None
assert kcsb.password is None
assert kcsb.application_client_id is None
assert kcsb.application_key is None
assert kcsb.user_token is None
assert kcsb.authority_id == "common"
assert repr(kcsb) == "Data Source=localhost;AAD Federated Security=True;Authority Id=common;Application Token=%s" % token
assert str(kcsb) == "Data Source=localhost;AAD Federated Security=True;Authority Id=common;Application Token=%s" % self.PASSWORDS_REPLACEMENT
def test_aad_user_token(self):
"""Checks kcsb that is created with AAD user token."""
token = "The user hardest token ever"
kcsb = KustoConnectionStringBuilder.with_aad_user_token_authentication("localhost", user_token=token)
assert kcsb.data_source == "localhost"
assert kcsb.user_token == token
assert kcsb.aad_federated_security
assert kcsb.aad_user_id is None
assert kcsb.password is None
assert kcsb.application_client_id is None
assert kcsb.application_key is None
assert kcsb.application_token is None
assert kcsb.authority_id == "common"
assert repr(kcsb) == "Data Source=localhost;AAD Federated Security=True;Authority Id=common;User Token=%s" % token
assert str(kcsb) == "Data Source=localhost;AAD Federated Security=True;Authority Id=common;User Token=%s" % self.PASSWORDS_REPLACEMENT
def test_add_msi(self):
client_guid = "kjhjk"
object_guid = "87687687"
res_guid = "kajsdghdijewhag"
"""
Use of object_id and msi_res_id is disabled pending support of azure-identity
When version 1.4.1 is released and these parameters are supported enable the functionality and tests back
"""
exception_occurred = False
try:
KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication("localhost2", object_id=object_guid, timeout=3)
except ValueError:
exception_occurred = True
assert exception_occurred is True
exception_occurred = False
try:
KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication("localhost3", msi_res_id=res_guid)
except ValueError:
exception_occurred = True
assert exception_occurred is True
kcsb = [
KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication("localhost0", timeout=1),
KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication("localhost1", client_id=client_guid, timeout=2),
# KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication("localhost2", object_id=object_guid, timeout=3),
# KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication("localhost3", msi_res_id=res_guid),
]
assert kcsb[0].msi_authentication
assert kcsb[0].msi_parameters["connection_timeout"] == 1
assert "client_id" not in kcsb[0].msi_parameters
assert "object_id" not in kcsb[0].msi_parameters
assert "msi_res_id" not in kcsb[0].msi_parameters
assert kcsb[1].msi_authentication
assert kcsb[1].msi_parameters["connection_timeout"] == 2
assert kcsb[1].msi_parameters["client_id"] == client_guid
assert "object_id" not in kcsb[1].msi_parameters
assert "msi_res_id" not in kcsb[1].msi_parameters
"""
assert kcsb[2].msi_authentication
assert kcsb[2].msi_parameters["connection_timeout"] == 3
assert "client_id" not in kcsb[2].msi_parameters
assert kcsb[2].msi_parameters["object_id"] == object_guid
assert "msi_res_id" not in kcsb[2].msi_parameters
assert kcsb[3].msi_authentication
assert "timeout" not in kcsb[3].msi_parameters
assert "client_id" not in kcsb[3].msi_parameters
assert "object_id" not in kcsb[3].msi_parameters
assert kcsb[3].msi_parameters["msi_res_id"] == res_guid
"""
exception_occurred = False
try:
fault = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication("localhost", client_id=client_guid, object_id=object_guid)
except ValueError as e:
exception_occurred = True
finally:
assert exception_occurred
def test_add_token_provider(self):
caller_token = "caller token"
token_provider = lambda: caller_token
kscb = KustoConnectionStringBuilder.with_token_provider("localhost", token_provider)
assert kscb.token_provider() == caller_token
exception_occurred = False
try:
kscb = KustoConnectionStringBuilder.with_token_provider("localhost", caller_token)
except AssertionError as ex:
exception_occurred = True
finally:
assert exception_occurred
def test_add_async_token_provider(self):
caller_token = "caller token"
async def async_token_provider():
return caller_token
kscb = KustoConnectionStringBuilder.with_async_token_provider("localhost", async_token_provider)
assert kscb.async_token_provider() is not None
exception_occurred = False
try:
kscb = KustoConnectionStringBuilder.with_async_token_provider("localhost", caller_token)
except AssertionError as ex:
exception_occurred = True
finally:
assert exception_occurred
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_models.py 0000664 0000000 0000000 00000000647 14172227630 0024574 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import json
import os
from azure.kusto.data._models import KustoResultTable
def test_str_and_dates_smoke():
with open(os.path.join(os.path.dirname(__file__), "input", "deft.json"), "r") as f:
data = f.read()
json_table = json.loads(data)[2]
result_table = KustoResultTable(json_table)
assert len(str(result_table)) == 4537
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_security.py 0000664 0000000 0000000 00000013141 14172227630 0025151 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import pytest
from azure.kusto.data import KustoConnectionStringBuilder
from azure.kusto.data._token_providers import *
from azure.kusto.data.exceptions import KustoAuthenticationError
from azure.kusto.data.security import _AadHelper
KUSTO_TEST_URI = "https://thisclusterdoesnotexist.kusto.windows.net"
TEST_INTERACTIVE_AUTH = False # User interaction required, enable this when running test manually
CloudSettings._cloud_cache[KUSTO_TEST_URI] = CloudSettings.DEFAULT_CLOUD
CloudSettings._cloud_cache["https://somecluster.kusto.windows.net"] = CloudSettings.DEFAULT_CLOUD
def test_unauthorized_exception():
"""Test the exception thrown when authorization fails."""
cluster = "https://somecluster.kusto.windows.net"
username = "username@microsoft.com"
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(cluster, username, "StrongestPasswordEver", "authorityName")
aad_helper = _AadHelper(kcsb, False)
aad_helper.token_provider._init_resources()
try:
aad_helper.acquire_authorization_header()
except KustoAuthenticationError as error:
assert error.authentication_method == UserPassTokenProvider.name()
assert error.authority == "https://login.microsoftonline.com/authorityName"
assert error.kusto_cluster == cluster
assert error.kwargs["username"] == username
assert error.kwargs["client_id"] == CloudSettings.DEFAULT_CLOUD.kusto_client_app_id
# TODO: remove this once we can control the timeout
@pytest.mark.skip()
def test_msi_auth():
"""
* * * Note * * *
Each connection test takes about 15-20 seconds which is the time it takes TCP to fail connecting to the nonexistent MSI endpoint
The timeout option does not seem to affect this behavior. Could be it only affects the waiting time fora response in successful connections.
Please be prudent in adding any future tests!
"""
client_guid = "kjhjk"
object_guid = "87687687"
res_guid = "kajsdghdijewhag"
"""
Use of object_id and msi_res_id is disabled pending support of azure-identity
When version 1.4.1 is released and these parameters are supported enable the functionality and tests back
"""
kcsb = [
KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(KUSTO_TEST_URI, timeout=1),
KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(KUSTO_TEST_URI, client_id=client_guid, timeout=1),
# KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(KUSTO_TEST_URI, object_id=object_guid, timeout=1),
# KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(KUSTO_TEST_URI, msi_res_id=res_guid, timeout=1),
]
helpers = [_AadHelper(i, False) for i in kcsb]
for h in helpers:
h.token_provider._init_resources()
try:
helpers[0].acquire_authorization_header()
except KustoAuthenticationError as e:
assert e.authentication_method == MsiTokenProvider.name()
assert "client_id" not in e.kwargs
assert "object_id" not in e.kwargs
assert "msi_res_id" not in e.kwargs
try:
helpers[1].acquire_authorization_header()
except KustoAuthenticationError as e:
assert e.authentication_method == MsiTokenProvider.name()
assert e.kwargs["client_id"] == client_guid
assert "object_id" not in e.kwargs
assert "msi_res_id" not in e.kwargs
assert str(e.exception).index("client_id") > -1
assert str(e.exception).index(client_guid) > -1
def test_token_provider_auth():
valid_token_provider = lambda: "caller token"
invalid_token_provider = lambda: 12345678
valid_kcsb = KustoConnectionStringBuilder.with_token_provider(KUSTO_TEST_URI, valid_token_provider)
invalid_kcsb = KustoConnectionStringBuilder.with_token_provider(KUSTO_TEST_URI, invalid_token_provider)
valid_helper = _AadHelper(valid_kcsb, False)
valid_helper.token_provider._init_resources()
invalid_helper = _AadHelper(invalid_kcsb, False)
invalid_helper.token_provider._init_resources()
auth_header = valid_helper.acquire_authorization_header()
assert auth_header.index(valid_token_provider()) > -1
try:
invalid_helper.acquire_authorization_header()
except KustoAuthenticationError as e:
assert e.authentication_method == CallbackTokenProvider.name()
assert str(e.exception).index(str(type(invalid_token_provider()))) > -1
def test_user_app_token_auth():
token = "123456446"
user_kcsb = KustoConnectionStringBuilder.with_aad_user_token_authentication(KUSTO_TEST_URI, token)
app_kcsb = KustoConnectionStringBuilder.with_aad_application_token_authentication(KUSTO_TEST_URI, token)
user_helper = _AadHelper(user_kcsb, False)
app_helper = _AadHelper(app_kcsb, False)
user_helper.token_provider._init_resources()
app_helper.token_provider._init_resources()
auth_header = user_helper.acquire_authorization_header()
assert auth_header.index(token) > -1
auth_header = app_helper.acquire_authorization_header()
assert auth_header.index(token) > -1
def test_interactive_login():
if not TEST_INTERACTIVE_AUTH:
print(" *** Skipped interactive login Test ***")
return
kcsb = KustoConnectionStringBuilder.with_interactive_login(KUSTO_TEST_URI)
aad_helper = _AadHelper(kcsb, False)
# should prompt
header = aad_helper.acquire_authorization_header()
assert header is not None
# should not prompt
header = aad_helper.acquire_authorization_header()
assert header is not None
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_streaming_query.py 0000664 0000000 0000000 00000031712 14172227630 0026524 0 ustar 00root root 0000000 0000000 import os
from io import StringIO
import pytest
from azure.kusto.data._models import WellKnownDataSet, KustoResultRow, KustoResultColumn
from azure.kusto.data.aio.response import KustoStreamingResponseDataSet as AsyncKustoStreamingResponseDataSet
from azure.kusto.data.aio.streaming_response import JsonTokenReader as AsyncJsonTokenReader, StreamingDataSetEnumerator as AsyncProgressiveDataSetEnumerator
from azure.kusto.data.exceptions import KustoServiceError, KustoStreamingQueryError, KustoTokenParsingError, KustoUnsupportedApiError, KustoMultiApiError
from azure.kusto.data.response import KustoStreamingResponseDataSet
from azure.kusto.data.streaming_response import JsonTokenReader, StreamingDataSetEnumerator, FrameType, JsonTokenType
from tests.kusto_client_common import KustoClientTestsMixin
class MockAioFile:
def __init__(self, filename):
self.filename = filename
def __enter__(self):
self.file = open(self.filename, "rb")
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.file.close()
async def read(self, n=-1):
return self.file.read(n)
class AsyncStringIO:
def __init__(self, string):
self.string_io = StringIO(string)
async def read(self, n=-1):
return self.string_io.read(n)
class TestStreamingQuery(KustoClientTestsMixin):
"""Tests class for KustoClient API"""
@staticmethod
def open_json_file(file_name: str):
return open(os.path.join(os.path.dirname(__file__), "input", file_name), "rb")
@staticmethod
def open_async_json_file(file_name: str):
return MockAioFile(os.path.join(os.path.dirname(__file__), "input", file_name))
def test_sanity(self):
with self.open_json_file("deft.json") as f:
reader = StreamingDataSetEnumerator(JsonTokenReader(f))
for i in reader:
if i["FrameType"] == FrameType.DataTable and i["TableKind"] == WellKnownDataSet.PrimaryResult.value:
columns = [KustoResultColumn(column, index) for index, column in enumerate(i["Columns"])]
self._assert_sanity_query_primary_results(KustoResultRow(columns, r) for r in i["Rows"])
def test_alternative_order(self):
with self.open_json_file("alternative_order.json") as f:
reader = StreamingDataSetEnumerator(JsonTokenReader(f))
for i in reader:
if i["FrameType"] == FrameType.DataTable and i["TableKind"] == WellKnownDataSet.PrimaryResult.value:
columns = [KustoResultColumn(column, index) for index, column in enumerate(i["Columns"])]
self._assert_sanity_query_primary_results(KustoResultRow(columns, r) for r in i["Rows"])
def test_progressive_unsupported(self):
with self.open_json_file("progressive_result.json") as f:
reader = StreamingDataSetEnumerator(JsonTokenReader(f))
with pytest.raises(KustoUnsupportedApiError):
for _ in reader:
pass
with self.open_json_file("deft_with_progressive_result.json") as f:
reader = StreamingDataSetEnumerator(JsonTokenReader(f))
with pytest.raises(KustoUnsupportedApiError):
for _ in reader:
pass
def test_dynamic(self):
with self.open_json_file("dynamic.json") as f:
reader = StreamingDataSetEnumerator(JsonTokenReader(f))
for i in reader:
if i["FrameType"] == FrameType.DataTable and i["TableKind"] == WellKnownDataSet.PrimaryResult.value:
row = next(i["Rows"])
self._assert_dynamic_response(row)
def test_sanity_kusto_streaming_response_dataset(self):
with self.open_json_file("deft.json") as f:
reader = StreamingDataSetEnumerator(JsonTokenReader(f))
response = KustoStreamingResponseDataSet(reader)
primary_tables = response.iter_primary_results()
# Before reading all of the tables these results won't be available
with pytest.raises(KustoStreamingQueryError):
errors_count = response.errors_count
with pytest.raises(KustoStreamingQueryError):
exceptions = response.get_exceptions()
first_table = next(primary_tables)
# Can't advance by default until current table is finished
with pytest.raises(KustoStreamingQueryError):
next(primary_tables)
self._assert_sanity_query_primary_results(first_table)
assert next(primary_tables, None) is None
assert response.errors_count == 0
assert response.get_exceptions() == []
assert response.finished
def test_exception_in_row(self):
with self.open_json_file("query_partial_results_defer_is_false.json") as f:
reader = StreamingDataSetEnumerator(JsonTokenReader(f))
response = KustoStreamingResponseDataSet(reader)
table = next(response.iter_primary_results())
with pytest.raises(KustoServiceError):
rows = [r for r in table]
@pytest.mark.asyncio
async def test_sanity_async(self):
with self.open_async_json_file("deft.json") as f:
reader = AsyncProgressiveDataSetEnumerator(AsyncJsonTokenReader(f))
async for i in reader:
if i["FrameType"] == FrameType.DataTable and i["TableKind"] == WellKnownDataSet.PrimaryResult.value:
columns = [KustoResultColumn(column, index) for index, column in enumerate(i["Columns"])]
rows = [KustoResultRow(columns, r) async for r in i["Rows"]]
self._assert_sanity_query_primary_results(rows)
@pytest.mark.asyncio
async def test_alternative_order_async(self):
with self.open_async_json_file("alternative_order.json") as f:
reader = AsyncProgressiveDataSetEnumerator(AsyncJsonTokenReader(f))
async for i in reader:
if i["FrameType"] == FrameType.DataTable and i["TableKind"] == WellKnownDataSet.PrimaryResult.value:
columns = [KustoResultColumn(column, index) for index, column in enumerate(i["Columns"])]
rows = [KustoResultRow(columns, r) async for r in i["Rows"]]
self._assert_sanity_query_primary_results(rows)
@pytest.mark.asyncio
async def test_progressive_unsupported_async(self):
with self.open_async_json_file("progressive_result.json") as f:
reader = AsyncProgressiveDataSetEnumerator(AsyncJsonTokenReader(f))
with pytest.raises(KustoUnsupportedApiError):
async for _ in reader:
pass
with self.open_async_json_file("deft_with_progressive_result.json") as f:
reader = AsyncProgressiveDataSetEnumerator(AsyncJsonTokenReader(f))
with pytest.raises(KustoUnsupportedApiError):
async for _ in reader:
pass
@pytest.mark.asyncio
async def test_dynamic_async(self):
with self.open_async_json_file("dynamic.json") as f:
reader = AsyncProgressiveDataSetEnumerator(AsyncJsonTokenReader(f))
async for i in reader:
if i["FrameType"] == FrameType.DataTable and i["TableKind"] == WellKnownDataSet.PrimaryResult.value:
row = await i["Rows"].__anext__()
self._assert_dynamic_response(row)
@pytest.mark.asyncio
async def test_sanity_kusto_streaming_response_dataset_async(self):
with self.open_async_json_file("deft.json") as f:
reader = AsyncProgressiveDataSetEnumerator(AsyncJsonTokenReader(f))
response = AsyncKustoStreamingResponseDataSet(reader)
primary_tables = response.iter_primary_results()
# Before reading all of the tables these results won't be available
with pytest.raises(KustoStreamingQueryError):
errors_count = response.errors_count
with pytest.raises(KustoStreamingQueryError):
exceptions = response.get_exceptions()
first_table = await primary_tables.__anext__()
# Can't advance by default until current table is finished
with pytest.raises(KustoStreamingQueryError):
await primary_tables.__anext__()
self._assert_sanity_query_primary_results([x async for x in first_table])
with pytest.raises(StopAsyncIteration):
await primary_tables.__anext__()
assert response.errors_count == 0
assert response.get_exceptions() == []
assert response.finished
@pytest.mark.asyncio
async def test_exception_in_row_async(self):
with self.open_async_json_file("query_partial_results_defer_is_false.json") as f:
reader = AsyncProgressiveDataSetEnumerator(AsyncJsonTokenReader(f))
response = AsyncKustoStreamingResponseDataSet(reader)
table = await response.iter_primary_results().__anext__()
with pytest.raises(KustoMultiApiError):
rows = [r async for r in table]
class TestJsonTokenReader:
def get_reader(self, data) -> JsonTokenReader:
return JsonTokenReader(StringIO(data))
def get_async_reader(self, data) -> AsyncJsonTokenReader:
return AsyncJsonTokenReader(AsyncStringIO(data))
def test_reading_token(self):
reader = self.get_reader("{")
assert reader.read_next_token_or_throw().token_type == JsonTokenType.START_MAP
with pytest.raises(KustoTokenParsingError):
reader.read_next_token_or_throw()
def test_read_token_of_type(self):
reader = self.get_reader("{}")
assert reader.read_token_of_type(JsonTokenType.START_MAP).token_type == JsonTokenType.START_MAP
with pytest.raises(KustoTokenParsingError):
reader.read_token_of_type(JsonTokenType.START_MAP)
def test_types(self):
reader = self.get_reader('{"junk": [[[]]], "key": ["a",false,3.63]}')
assert reader.read_start_object().token_type == JsonTokenType.START_MAP
assert reader.skip_until_property_name("key").token_type == JsonTokenType.MAP_KEY
assert reader.read_start_array().token_type == JsonTokenType.START_ARRAY
assert reader.read_string() == "a"
assert reader.read_boolean() is False
assert reader.read_number() == 3.63
def test_skip(self):
reader = self.get_reader('{"junk": [[[]]], "key": "a", "junk2": {"key2": "aaaaa"}, "key2": "www"}')
assert reader.read_start_object().token_type == JsonTokenType.START_MAP
key = reader.skip_until_any_property_name("key", "junk2")
assert key.token_type == JsonTokenType.MAP_KEY
assert reader.read_string() == "a"
key2 = reader.skip_until_property_name_or_end_object("key2")
assert key2.token_type == JsonTokenType.MAP_KEY
assert key2.token_path == ""
assert reader.read_string() == "www"
assert reader.skip_until_property_name_or_end_object().token_type == JsonTokenType.END_MAP
@pytest.mark.asyncio
async def test_reading_token_async(self):
reader = self.get_async_reader("{")
assert (await reader.read_next_token_or_throw()).token_type == JsonTokenType.START_MAP
with pytest.raises(KustoTokenParsingError):
await reader.read_next_token_or_throw()
@pytest.mark.asyncio
async def test_read_token_of_type_async(self):
reader = self.get_async_reader("{}")
assert (await reader.read_token_of_type(JsonTokenType.START_MAP)).token_type == JsonTokenType.START_MAP
with pytest.raises(KustoTokenParsingError):
await reader.read_token_of_type(JsonTokenType.START_MAP)
@pytest.mark.asyncio
async def test_types_async(self):
reader = self.get_async_reader('{"junk": [[[]]], "key": ["a",false,3.63]}')
assert (await reader.read_start_object()).token_type == JsonTokenType.START_MAP
assert (await reader.skip_until_property_name("key")).token_type == JsonTokenType.MAP_KEY
assert (await reader.read_start_array()).token_type == JsonTokenType.START_ARRAY
assert (await reader.read_string()) == "a"
assert (await reader.read_boolean()) is False
assert (await reader.read_number()) == 3.63
@pytest.mark.asyncio
async def test_skip_async(self):
reader = self.get_async_reader('{"junk": [[[]]], "key": "a", "junk2": {"key2": "aaaaa"}, "key2": "www" }')
assert (await reader.read_start_object()).token_type == JsonTokenType.START_MAP
key = await reader.skip_until_any_property_name("key", "junk2")
assert key.token_type == JsonTokenType.MAP_KEY
assert (await reader.read_string()) == "a"
key2 = await reader.skip_until_property_name_or_end_object("key2")
assert key2.token_type == JsonTokenType.MAP_KEY
assert key2.token_path == ""
assert (await reader.read_string()) == "www"
assert (await reader.skip_until_property_name_or_end_object()).token_type == JsonTokenType.END_MAP
azure-kusto-python-3.0.1/azure-kusto-data/tests/test_token_providers.py 0000664 0000000 0000000 00000033107 14172227630 0026523 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import os
import unittest
from threading import Thread
from asgiref.sync import async_to_sync
from azure.kusto.data._cloud_settings import CloudInfo
from azure.kusto.data._token_providers import *
KUSTO_URI = "https://sdkse2etest.eastus.kusto.windows.net"
TOKEN_VALUE = "little miss sunshine"
TEST_AZ_AUTH = False # enable this in environments with az cli installed, and make sure to call 'az login' first
TEST_MSI_AUTH = False # enable this in environments with MSI enabled and make sure to set the relevant environment variables
TEST_DEVICE_AUTH = False # User interaction required, enable this when running test manually
TEST_INTERACTIVE_AUTH = False # User interaction required, enable this when running test manually
class MockProvider(TokenProviderBase):
def __init__(self, is_async: bool = False):
super().__init__(is_async)
self._silent_token = False
self.init_count = 0
@staticmethod
def name() -> str:
return "MockProvider"
def _context_impl(self) -> dict:
return {"authority": "MockProvider"}
def _init_impl(self):
self.init_count = self.init_count + 1
def _get_token_impl(self) -> Optional[dict]:
self._silent_token = True
return {TokenConstants.MSAL_ACCESS_TOKEN: "token"}
def _get_token_from_cache_impl(self) -> Optional[dict]:
if self._silent_token:
return {TokenConstants.MSAL_ACCESS_TOKEN: "token"}
return None
class TokenProviderTests(unittest.TestCase):
@staticmethod
def test_base_provider():
# test init with no URI
provider = MockProvider()
# Test provider with URI, No silent token
provider = MockProvider()
token = provider._get_token_from_cache_impl()
assert provider.init_count == 0
assert token is None
token = provider.get_token()
assert provider.init_count == 1
assert TokenConstants.MSAL_ACCESS_TOKEN in token
token = provider._get_token_from_cache_impl()
assert TokenConstants.MSAL_ACCESS_TOKEN in token
token = provider.get_token()
assert provider.init_count == 1
good_token = {TokenConstants.MSAL_ACCESS_TOKEN: TOKEN_VALUE}
bad_token1 = None
bad_token2 = {"error": "something bad occurred"}
assert provider._valid_token_or_none(good_token) == good_token
assert provider._valid_token_or_none(bad_token1) is None
assert provider._valid_token_or_none(bad_token2) is None
assert provider._valid_token_or_throw(good_token) == good_token
exception_occurred = False
try:
provider._valid_token_or_throw(bad_token1)
except KustoClientError:
exception_occurred = True
finally:
assert exception_occurred
exception_occurred = False
try:
provider._valid_token_or_throw(bad_token2)
except KustoClientError:
exception_occurred = True
finally:
assert exception_occurred
@staticmethod
def get_token_value(token: dict):
assert token is not None
assert TokenConstants.MSAL_ERROR not in token
value = None
if TokenConstants.MSAL_ACCESS_TOKEN in token:
return token[TokenConstants.MSAL_ACCESS_TOKEN]
elif TokenConstants.AZ_ACCESS_TOKEN in token:
return token[TokenConstants.AZ_ACCESS_TOKEN]
else:
assert False
@staticmethod
def test_fail_async_call():
provider = BasicTokenProvider(token=TOKEN_VALUE)
try:
async_to_sync(provider.get_token_async)()
assert False, "Expected KustoAsyncUsageError to occur"
except KustoAsyncUsageError as e:
assert str(e) == "Method get_token_async can't be called from a synchronous client"
try:
async_to_sync(provider.context_async)()
assert False, "Expected KustoAsyncUsageError to occur"
except KustoAsyncUsageError as e:
assert str(e) == "Method context_async can't be called from a synchronous client"
@staticmethod
def test_basic_provider():
provider = BasicTokenProvider(token=TOKEN_VALUE)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) == TOKEN_VALUE
@staticmethod
def test_basic_provider_in_thread():
exc = []
def inner(exc):
try:
TokenProviderTests.test_basic_provider()
except Exception as e:
exc.append(e)
pass
t = Thread(target=inner, args=(exc,))
t.start()
t.join()
if exc:
raise exc[0]
@staticmethod
def test_callback_token_provider():
provider = CallbackTokenProvider(token_callback=lambda: TOKEN_VALUE, async_token_callback=None)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) == TOKEN_VALUE
provider = CallbackTokenProvider(token_callback=lambda: 0, async_token_callback=None) # token is not a string
exception_occurred = False
try:
provider.get_token()
except KustoClientError:
exception_occurred = True
finally:
assert exception_occurred
@staticmethod
def test_az_provider():
if not TEST_AZ_AUTH:
print(" *** Skipped Az-Cli Provider Test ***")
return
print("Note!\nThe test 'test_az_provider' will fail if 'az login' was not called.")
provider = AzCliTokenProvider(KUSTO_URI)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
# another run to pass through the cache
token = provider._get_token_from_cache_impl()
assert TokenProviderTests.get_token_value(token) is not None
@staticmethod
def test_msi_provider():
if not TEST_MSI_AUTH:
print(" *** Skipped MSI Provider Test ***")
return
user_msi_object_id = os.environ.get("MSI_OBJECT_ID")
user_msi_client_id = os.environ.get("MSI_CLIENT_ID")
# system MSI
provider = MsiTokenProvider(KUSTO_URI)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
if user_msi_object_id is not None:
args = {"object_id": user_msi_object_id}
provider = MsiTokenProvider(KUSTO_URI, args)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
else:
print(" *** Skipped MSI Provider Client Id Test ***")
if user_msi_client_id is not None:
args = {"client_id": user_msi_client_id}
provider = MsiTokenProvider(KUSTO_URI, args)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
else:
print(" *** Skipped MSI Provider Object Id Test ***")
@staticmethod
def test_user_pass_provider():
username = os.environ.get("USER_NAME")
password = os.environ.get("USER_PASS")
auth = os.environ.get("USER_AUTH_ID", "organizations")
if username and password and auth:
provider = UserPassTokenProvider(KUSTO_URI, auth, username, password)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert TokenProviderTests.get_token_value(token) is not None
else:
print(" *** Skipped User & Pass Provider Test ***")
@staticmethod
def test_device_auth_provider():
if not TEST_DEVICE_AUTH:
print(" *** Skipped User Device Flow Test ***")
return
def callback(x):
# break here if you debug this test, and get the code from 'x'
print(x)
provider = DeviceLoginTokenProvider(KUSTO_URI, "organizations", callback)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert TokenProviderTests.get_token_value(token) is not None
@staticmethod
def test_interactive_login():
if not TEST_INTERACTIVE_AUTH:
print(" *** Skipped interactive login Test ***")
return
auth_id = os.environ.get("APP_AUTH_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47")
provider = InteractiveLoginTokenProvider(KUSTO_URI, auth_id)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert TokenProviderTests.get_token_value(token) is not None
@staticmethod
def test_app_key_provider():
# default details are for kusto-client-e2e-test-app
# to run the test, get the key from Azure portal
app_id = os.environ.get("APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f")
auth_id = os.environ.get("APP_AUTH_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47")
app_key = os.environ.get("APP_KEY")
if app_id and app_key and auth_id:
provider = ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert TokenProviderTests.get_token_value(token) is not None
else:
print(" *** Skipped App Id & Key Provider Test ***")
@staticmethod
def test_app_cert_provider():
# default details are for kusto-client-e2e-test-app
# to run the test download the certs from Azure Portal
cert_app_id = os.environ.get("CERT_APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f")
cert_auth = os.environ.get("CERT_AUTH", "72f988bf-86f1-41af-91ab-2d7cd011db47")
thumbprint = os.environ.get("CERT_THUMBPRINT")
public_cert_path = os.environ.get("PUBLIC_CERT_PATH")
pem_key_path = os.environ.get("CERT_PEM_KEY_PATH")
if pem_key_path and thumbprint and cert_app_id:
with open(pem_key_path, "rb") as file:
pem_key = file.read()
provider = ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert TokenProviderTests.get_token_value(token) is not None
if public_cert_path:
with open(public_cert_path, "r") as file:
public_cert = file.read()
provider = ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint, public_cert)
token = provider.get_token()
assert TokenProviderTests.get_token_value(token) is not None
# Again through cache
token = provider._get_token_from_cache_impl()
assert TokenProviderTests.get_token_value(token) is not None
else:
print(" *** Skipped App Cert SNI Provider Test ***")
else:
print(" *** Skipped App Cert Provider Test ***")
@staticmethod
def test_cloud_mfa_off():
FAKE_URI = "https://fake_cluster_for_login_mfa_test.kusto.windows.net"
cloud = CloudInfo(
login_endpoint="https://login_endpoint",
login_mfa_required=False,
kusto_client_app_id="1234",
kusto_client_redirect_uri="",
kusto_service_resource_id="https://fakeurl.kusto.windows.net",
first_party_authority_url="",
)
CloudSettings._cloud_cache[FAKE_URI] = cloud
authority = "auth_test"
provider = UserPassTokenProvider(FAKE_URI, authority, "a", "b")
provider._init_once(init_only_resources=True)
context = provider.context()
assert context["authority"] == "https://login_endpoint/auth_test"
assert context["client_id"] == cloud.kusto_client_app_id
assert provider._scopes == ["https://fakeurl.kusto.windows.net/.default"]
@staticmethod
def test_cloud_mfa_on():
FAKE_URI = "https://fake_cluster_for_login_mfa_test.kusto.windows.net"
cloud = CloudInfo(
login_endpoint="https://login_endpoint",
login_mfa_required=True,
kusto_client_app_id="1234",
kusto_client_redirect_uri="",
kusto_service_resource_id="https://fakeurl.kusto.windows.net",
first_party_authority_url="",
)
CloudSettings._cloud_cache[FAKE_URI] = cloud
authority = "auth_test"
provider = UserPassTokenProvider(FAKE_URI, authority, "a", "b")
provider._init_once(init_only_resources=True)
context = provider.context()
assert context["authority"] == "https://login_endpoint/auth_test"
assert context["client_id"] == "1234"
assert provider._scopes == ["https://fakeurl.kustomfa.windows.net/.default"]
azure-kusto-python-3.0.1/azure-kusto-data/tox.ini 0000664 0000000 0000000 00000000133 14172227630 0022037 0 ustar 00root root 0000000 0000000 [tox]
envlist = py27,py35,py37
[testenv]
deps=mock
pytest
pandas
commands = pytest azure-kusto-python-3.0.1/azure-kusto-ingest/ 0000775 0000000 0000000 00000000000 14172227630 0021107 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-ingest/README.rst 0000664 0000000 0000000 00000003625 14172227630 0022604 0 ustar 00root root 0000000 0000000 Microsoft Azure Kusto Ingest Library for Python
===============================================
.. code-block:: python
from azure.kusto.data import KustoConnectionStringBuilder, DataFormat
from azure.kusto.ingest import QueuedIngestClient, IngestionProperties, FileDescriptor, BlobDescriptor
ingestion_props = IngestionProperties(database="{database_name}", table="{table_name}", data_format=DataFormat.CSV)
client = QueuedIngestClient(KustoConnectionStringBuilder.with_interactive_login("https://ingest-{cluster_name}.kusto.windows.net"))
file_descriptor = FileDescriptor("{filename}.csv", 15360) # in this example, the raw (uncompressed) size of the data is 15KB (15360 bytes)
client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_props)
client.ingest_from_file("{filename}.csv", ingestion_properties=ingestion_props)
blob_descriptor = BlobDescriptor("https://{path_to_blob}.csv.gz?sas", 51200) # in this example, the raw (uncompressed) size of the data is 50KB (52100 bytes)
client.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_props)
Overview
--------
*Kusto Python Ingest Client* Library provides the capability to ingest data into Kusto clusters using Python.
It is Python 3.x compatible and supports data types through familiar Python DB API interface.
It's possible to use the library, for instance, from `Jupyter Notebooks `_ which are attached to Spark clusters,
including, but not exclusively, `Azure Databricks `_ instances.
* `How to install the package `_.
* `Data ingest sample `_.
* `GitHub Repository `_.
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/ 0000775 0000000 0000000 00000000000 14172227630 0022235 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-ingest/azure/__init__.py 0000664 0000000 0000000 00000000361 14172227630 0024346 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
"""Init file for azure namespace. https://github.com/Azure/azure-sdk-for-python/wiki/Azure-packaging"""
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ 0000775 0000000 0000000 00000000000 14172227630 0023402 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/__init__.py 0000664 0000000 0000000 00000000361 14172227630 0025513 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
"""Init file for azure namespace. https://github.com/Azure/azure-sdk-for-python/wiki/Azure-packaging"""
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/ 0000775 0000000 0000000 00000000000 14172227630 0024673 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/__init__.py 0000664 0000000 0000000 00000001414 14172227630 0027004 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from ._version import VERSION as __version__
from .base_ingest_client import IngestionResult, IngestionStatus
from .descriptors import BlobDescriptor, FileDescriptor, StreamDescriptor
from .exceptions import KustoMissingMappingError
from .ingest_client import QueuedIngestClient
from .ingestion_properties import (
ValidationPolicy,
ValidationImplications,
ValidationOptions,
ReportLevel,
ReportMethod,
IngestionProperties,
IngestionMappingKind,
ColumnMapping,
TransformationMethod,
)
from .managed_streaming_ingest_client import ManagedStreamingIngestClient
from .streaming_ingest_client import KustoStreamingIngestClient
from .base_ingest_client import BaseIngestClient
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py 0000664 0000000 0000000 00000014011 14172227630 0030722 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import re
from datetime import datetime, timedelta
from typing import List
from azure.kusto.data import KustoClient
from azure.kusto.data._models import KustoResultTable
_URI_FORMAT = re.compile("https://(\\w+).(queue|blob|table).(core.\\w+.\\w+)/([\\w,-]+)\\?(.*)")
_SHOW_VERSION = ".show version"
_SERVICE_TYPE_COLUMN_NAME = "ServiceType"
class _ResourceUri:
def __init__(self, storage_account_name: str, object_type: str, endpoint_suffix: str, object_name: str, sas: str):
self.storage_account_name = storage_account_name
self.object_type = object_type
self.endpoint_suffix = endpoint_suffix
self.object_name = object_name
self.sas = sas
@classmethod
def parse(cls, uri):
"""Parses uri into a ResourceUri object"""
match = _URI_FORMAT.search(uri)
return cls(match.group(1), match.group(2), match.group(3), match.group(4), match.group(5))
@property
def uri(self) -> str:
return "https://{0.storage_account_name}.{0.object_type}.{0.endpoint_suffix}/{0.object_name}".format(self)
@property
def account_uri(self) -> str:
return "https://{0.storage_account_name}.{0.object_type}.{0.endpoint_suffix}/?{0.sas}".format(self)
def __str__(self):
return "https://{0.storage_account_name}.{0.object_type}.{0.endpoint_suffix}/{0.object_name}?{0.sas}"
class _IngestClientResources:
def __init__(
self,
secured_ready_for_aggregation_queues: List[_ResourceUri] = None,
failed_ingestions_queues: List[_ResourceUri] = None,
successful_ingestions_queues: List[_ResourceUri] = None,
containers: List[_ResourceUri] = None,
status_tables: List[_ResourceUri] = None,
):
self.secured_ready_for_aggregation_queues = secured_ready_for_aggregation_queues
self.failed_ingestions_queues = failed_ingestions_queues
self.successful_ingestions_queues = successful_ingestions_queues
self.containers = containers
self.status_tables = status_tables
def is_applicable(self):
resources = [
self.secured_ready_for_aggregation_queues,
self.failed_ingestions_queues,
self.failed_ingestions_queues,
self.containers,
self.status_tables,
]
return all(resources)
class _ResourceManager:
def __init__(self, kusto_client: KustoClient):
self._kusto_client = kusto_client
self._refresh_period = timedelta(hours=1)
self._ingest_client_resources = None
self._ingest_client_resources_last_update = None
self._authorization_context = None
self._authorization_context_last_update = None
def _refresh_ingest_client_resources(self):
if (
not self._ingest_client_resources
or (self._ingest_client_resources_last_update + self._refresh_period) <= datetime.utcnow()
or not self._ingest_client_resources.is_applicable()
):
self._ingest_client_resources = self._get_ingest_client_resources_from_service()
self._ingest_client_resources_last_update = datetime.utcnow()
def _get_resource_by_name(self, table: KustoResultTable, resource_name: str):
return [_ResourceUri.parse(row["StorageRoot"]) for row in table if row["ResourceTypeName"] == resource_name]
def _get_ingest_client_resources_from_service(self):
table = self._kusto_client.execute("NetDefaultDB", ".get ingestion resources").primary_results[0]
secured_ready_for_aggregation_queues = self._get_resource_by_name(table, "SecuredReadyForAggregationQueue")
failed_ingestions_queues = self._get_resource_by_name(table, "FailedIngestionsQueue")
successful_ingestions_queues = self._get_resource_by_name(table, "SuccessfulIngestionsQueue")
containers = self._get_resource_by_name(table, "TempStorage")
status_tables = self._get_resource_by_name(table, "IngestionsStatusTable")
return _IngestClientResources(secured_ready_for_aggregation_queues, failed_ingestions_queues, successful_ingestions_queues, containers, status_tables)
def _refresh_authorization_context(self):
if (
not self._authorization_context
or self._authorization_context.isspace()
or (self._authorization_context_last_update + self._refresh_period) <= datetime.utcnow()
):
self._authorization_context = self._get_authorization_context_from_service()
self._authorization_context_last_update = datetime.utcnow()
def _get_authorization_context_from_service(self):
return self._kusto_client.execute("NetDefaultDB", ".get kusto identity token").primary_results[0][0]["AuthorizationContext"]
def get_ingestion_queues(self) -> List[_ResourceUri]:
self._refresh_ingest_client_resources()
return self._ingest_client_resources.secured_ready_for_aggregation_queues
def get_failed_ingestions_queues(self) -> List[_ResourceUri]:
self._refresh_ingest_client_resources()
return self._ingest_client_resources.failed_ingestions_queues
def get_successful_ingestions_queues(self) -> List[_ResourceUri]:
self._refresh_ingest_client_resources()
return self._ingest_client_resources.successful_ingestions_queues
def get_containers(self) -> List[_ResourceUri]:
self._refresh_ingest_client_resources()
return self._ingest_client_resources.containers
def get_ingestions_status_tables(self) -> List[_ResourceUri]:
self._refresh_ingest_client_resources()
return self._ingest_client_resources.status_tables
def get_authorization_context(self):
self._refresh_authorization_context()
return self._authorization_context
def retrieve_service_type(self):
try:
command_result = self._kusto_client.execute("NetDefaultDB", _SHOW_VERSION)
return command_result.primary_results[0][0][_SERVICE_TYPE_COLUMN_NAME]
except (TypeError, KeyError):
return ""
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/_retry.py 0000664 0000000 0000000 00000001110 14172227630 0026542 0 ustar 00root root 0000000 0000000 import random
from time import sleep
class ExponentialRetry:
def __init__(self, max_attempts, sleep_base_secs: float = 1.0, max_jitter_secs: float = 1.0):
self.max_attempts = max_attempts
self.sleep_base_secs = sleep_base_secs
self.max_jitter_secs = max_jitter_secs
self.current_attempt = 0
def do_backoff(self):
sleep((self.sleep_base_secs * (2 ** self.current_attempt)) + (random.uniform(0, self.max_jitter_secs)))
self.current_attempt += 1
def __bool__(self):
return self.current_attempt < self.max_attempts
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/_status_q.py 0000664 0000000 0000000 00000011337 14172227630 0027254 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import random
from typing import List, Callable
from azure.kusto.ingest._resource_manager import _ResourceUri
from azure.storage.queue import QueueServiceClient, QueueClient, QueueMessage, TextBase64EncodePolicy, TextBase64DecodePolicy
class QueueDetails:
def __init__(self, name, service):
self.name = name
self.service = service
def __str__(self):
return "QueueDetails({0.name})".format(self)
class StatusQueue:
"""StatusQueue is a class to simplify access to Kusto status queues (backed by azure storage queues)."""
def __init__(self, get_queues_func: Callable[[], List[_ResourceUri]], message_cls):
self.get_queues_func = get_queues_func
self.message_cls = message_cls
def _get_queues(self) -> List[QueueClient]:
return [
QueueServiceClient(q.account_uri).get_queue_client(queue=q.object_name, message_decode_policy=TextBase64DecodePolicy())
for q in self.get_queues_func()
]
def is_empty(self) -> bool:
"""Checks if Status queue has any messages"""
return len(self.peek(1, raw=True)) == 0
def _deserialize_message(self, m: QueueMessage):
"""Deserialize a message and return at as `message_cls`
:param m: original message m.
"""
return self.message_cls(m.content)
# TODO: current implementation takes a union top n / len(queues), which is not ideal,
# because the user is not supposed to know that there can be multiple underlying queues
def peek(self, n=1, raw=False) -> List[QueueMessage]:
"""Peek status queue
:param int n: number of messages to return as part of peek.
:param bool raw: should message content be returned as is (no parsing).
"""
def _peek_specific_q(_q: QueueClient, _n: int) -> bool:
has_messages = False
for m in _q.peek_messages(max_messages=_n):
if m:
has_messages = True
result.append(m if raw else self._deserialize_message(m))
# short circuit to prevent unneeded work
if len(result) == n:
return True
return has_messages
queues = self._get_queues()
random.shuffle(queues)
per_q = int(n / len(queues)) + 1
result = []
non_empty_qs = []
for q in queues:
if _peek_specific_q(q, per_q):
non_empty_qs.append(q)
if len(result) == n:
return result
# in-case queues aren't balanced, and we didn't get enough messages, iterate again and this time get all that we can
for q in non_empty_qs:
_peek_specific_q(q, n)
if len(result) == n:
return result
# because we ask for n / len(qs) + 1, we might get more message then requests
return result
# TODO: current implementation takes a union top n / len(queues), which is not ideal,
# because the user is not supposed to know that there can be multiple underlying queues
def pop(self, n: int = 1, raw: bool = False, delete: bool = True) -> List[QueueMessage]:
"""Pop status queue
:param int n: number of messages to return as part of peek.
:param bool raw: should message content be returned as is (no parsing).
:param bool delete: should message be deleted after pop. default is True as this is expected of a q.
"""
def _pop_specific_q(_q: QueueClient, _n: int) -> bool:
has_messages = False
for m in _q.receive_messages(messages_per_page=_n):
if m:
has_messages = True
result.append(m if raw else self._deserialize_message(m))
if delete:
_q.delete_message(m.id, m.pop_receipt)
# short circuit to prevent unneeded work
if len(result) == n:
return True
return has_messages
queues = self._get_queues()
random.shuffle(queues)
per_q = int(n / len(queues)) + 1
result = []
non_empty_qs = []
for q in queues:
if _pop_specific_q(q, per_q):
non_empty_qs.append(q)
if len(result) == n:
return result
# in-case queues aren't balanced, and we didn't get enough messages, iterate again and this time get all that we can
for q in non_empty_qs:
_pop_specific_q(q, n)
if len(result) == n:
return result
# because we ask for n / len(qs) + 1, we might get more message then requests
return result
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/_stream_extensions.py 0000664 0000000 0000000 00000004521 14172227630 0031160 0 ustar 00root root 0000000 0000000 import io
from typing import IO, AnyStr
def read_until_size_or_end(stream: IO[AnyStr], size: int) -> io.BytesIO:
pos = 0
result = io.BytesIO()
while True:
try:
returned = stream.read(size - pos)
pos += len(returned)
result.write(returned)
if len(returned) == 0 or pos == size:
result.seek(0, io.SEEK_SET)
return result
except BlockingIOError:
continue
class ChainStream(io.RawIOBase):
"""
https://stackoverflow.com/questions/24528278/stream-multiple-files-into-a-readable-object-in-python
"""
def __init__(self, streams):
self.leftover = b""
self.stream_iter = iter(streams)
try:
self.stream = next(self.stream_iter)
except StopIteration:
self.stream = None
def readable(self):
return True
def _read_next_chunk(self, max_length):
# Return 0 or more bytes from the current stream, first returning all
# leftover bytes. If the stream is closed returns b''
if self.leftover:
return self.leftover
elif self.stream is not None:
return self.stream.read(max_length)
else:
return b""
def readinto(self, b):
buffer_length = len(b)
chunk = self._read_next_chunk(buffer_length)
while len(chunk) == 0:
# move to next stream
if self.stream is not None:
self.stream.close()
try:
self.stream = next(self.stream_iter)
chunk = self._read_next_chunk(buffer_length)
except StopIteration:
# No more streams to chain together
self.stream = None
return 0 # indicate EOF
output, self.leftover = chunk[:buffer_length], chunk[buffer_length:]
b[: len(output)] = output
return len(output)
def chain_streams(streams, buffer_size=io.DEFAULT_BUFFER_SIZE):
"""
Chain an iterable of streams together into a single buffered stream.
Usage:
def generate_open_file_streams():
for file in filenames:
yield open(file, 'rb')
f = chain_streams(generate_open_file_streams())
f.read()
"""
return io.BufferedReader(ChainStream(streams), buffer_size=buffer_size)
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/_version.py 0000664 0000000 0000000 00000000132 14172227630 0027065 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
VERSION = "3.0.1"
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/base_ingest_client.py 0000664 0000000 0000000 00000011405 14172227630 0031067 0 ustar 00root root 0000000 0000000 import os
import tempfile
import time
import uuid
from abc import ABCMeta, abstractmethod
from copy import copy
from enum import Enum
from gzip import GzipFile
from io import TextIOWrapper, BytesIO
from typing import TYPE_CHECKING, Union, IO, AnyStr, Optional
from azure.kusto.data.data_format import DataFormat
from .descriptors import FileDescriptor, StreamDescriptor
from .ingestion_properties import IngestionProperties
if TYPE_CHECKING:
import pandas
class IngestionStatus(Enum):
"""
The ingestion was queued.
"""
QUEUED = "QUEUED"
"""
The ingestion was successfully streamed
"""
SUCCESS = "SUCCESS"
class IngestionResult:
"""
The result of an ingestion.
"""
status: IngestionStatus
"Will be `Queued` if the ingestion is queued, or `Success` if the ingestion is streaming and successful."
database: str
"The name of the database where the ingestion was performed."
table: str
"The name of the table where the ingestion was performed."
source_id: uuid.UUID
"The source id of the ingestion."
blob_uri: Optional[str]
"The blob uri of the ingestion, if exists."
def __init__(self, status: IngestionStatus, database: str, table: str, source_id: uuid.UUID, blob_uri: Optional[str] = None):
self.status = status
self.database = database
self.table = table
self.source_id = source_id
self.blob_uri = blob_uri
def __repr__(self):
return f"IngestionResult(status={self.status}, database={self.database}, table={self.table}, source_id={self.source_id}, blob_uri={self.blob_uri})"
class BaseIngestClient(metaclass=ABCMeta):
@abstractmethod
def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult:
"""Ingest from local files.
:param file_descriptor: a FileDescriptor to be ingested.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
pass
@abstractmethod
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
"""Ingest from io streams.
:param stream_descriptor: An object that contains a description of the stream to be ingested.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
pass
def ingest_from_dataframe(self, df: "pandas.DataFrame", ingestion_properties: IngestionProperties) -> IngestionResult:
"""Enqueue an ingest command from local files.
To learn more about ingestion methods go to:
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
:param pandas.DataFrame df: input dataframe to ingest.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
from pandas import DataFrame
if not isinstance(df, DataFrame):
raise ValueError("Expected DataFrame instance, found {}".format(type(df)))
file_name = "df_{id}_{timestamp}_{uid}.csv.gz".format(id=id(df), timestamp=int(time.time()), uid=uuid.uuid4())
temp_file_path = os.path.join(tempfile.gettempdir(), file_name)
df.to_csv(temp_file_path, index=False, encoding="utf-8", header=False, compression="gzip")
ingestion_properties.format = DataFormat.CSV
try:
return self.ingest_from_file(temp_file_path, ingestion_properties)
finally:
os.unlink(temp_file_path)
@staticmethod
def _prepare_stream(stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> StreamDescriptor:
if not isinstance(stream_descriptor, StreamDescriptor):
new_descriptor = StreamDescriptor(stream_descriptor)
else:
new_descriptor = copy(stream_descriptor)
if isinstance(new_descriptor.stream, TextIOWrapper):
stream = new_descriptor.stream.buffer
else:
stream = new_descriptor.stream
if not new_descriptor.is_compressed and ingestion_properties.format.compressible:
zipped_stream = BytesIO()
buffer = stream.read()
with GzipFile(filename="data", fileobj=zipped_stream, mode="wb") as f_out:
if isinstance(buffer, str):
data = bytes(buffer, "utf-8")
f_out.write(data)
else:
f_out.write(buffer)
zipped_stream.seek(0)
new_descriptor.is_compressed = True
new_descriptor.stream_name += ".gz"
stream = zipped_stream
new_descriptor.stream = stream
return new_descriptor
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/descriptors.py 0000664 0000000 0000000 00000013433 14172227630 0027612 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import os
import shutil
import struct
import uuid
from gzip import GzipFile
from io import BytesIO, SEEK_END
from typing import Union, Optional, AnyStr, IO
from zipfile import ZipFile
OptionalUUID = Optional[Union[str, uuid.UUID]]
def ensure_uuid(maybe_uuid: OptionalUUID) -> uuid.UUID:
if not maybe_uuid:
return uuid.uuid4()
if type(maybe_uuid) == uuid.UUID:
return maybe_uuid
return uuid.UUID(f"{maybe_uuid}", version=4)
class FileDescriptor:
"""FileDescriptor is used to describe a file that will be used as an ingestion source."""
# Gzip keeps the decompressed stream size as a UINT32 in the last 4 bytes of the stream, however this poses a limit to the expressed size which is 4GB
# The standard says that when the size is bigger then 4GB, the UINT rolls over.
# The below constant expresses the maximal size of a compressed stream that will not cause the UINT32 to rollover given a maximal compression ratio of 1:40
GZIP_MAX_DISK_SIZE_FOR_DETECTION = int(4 * 1024 * 1024 * 1024 / 40)
DEFAULT_COMPRESSION_RATIO = 11
def __init__(self, path: str, size: Optional[int] = None, source_id: OptionalUUID = None):
"""
:param path: file path.
:type path: str.
:param size: estimated size of file if known. if None or 0 will try to guess.
:type size: Optional[int].
:param source_id: a v4 uuid to serve as the source's id.
:type source_id: OptionalUUID
"""
self.path: str = path
self._size: Optional[int] = size
self._detect_size_once: bool = not size
self.source_id: uuid.UUID = ensure_uuid(source_id)
self.stream_name: str = os.path.basename(self.path)
@property
def size(self) -> int:
if self._detect_size_once:
self._detect_size()
self._detect_size_once = False
return self._size
@size.setter
def size(self, size: int):
if size:
self._size = size
self._detect_size_once = False
def _detect_size(self):
uncompressed_size = 0
if self.path.endswith(".gz"):
# This logic follow after the C# implementation
# See IngstionHelpers.cs for an explanation as to what stands behind it
with open(self.path, "rb") as f:
disk_size = f.seek(-4, SEEK_END)
uncompressed_size = struct.unpack("I", f.read(4))[0]
if (disk_size >= uncompressed_size) or (disk_size >= self.GZIP_MAX_DISK_SIZE_FOR_DETECTION):
uncompressed_size = disk_size * self.DEFAULT_COMPRESSION_RATIO
elif self.path.endswith(".zip"):
with ZipFile(self.path) as zip_archive:
for f in zip_archive.infolist():
uncompressed_size += f.file_size
else:
uncompressed_size = os.path.getsize(self.path)
self._size = uncompressed_size
@property
def is_compressed(self) -> bool:
return self.path.endswith(".gz") or self.path.endswith(".zip")
def open(self, should_compress: bool) -> BytesIO:
if should_compress:
self.stream_name += ".gz"
file_stream = BytesIO()
with open(self.path, "rb") as f_in, GzipFile(filename="data", fileobj=file_stream, mode="wb") as f_out:
shutil.copyfileobj(f_in, f_out)
file_stream.seek(0)
else:
file_stream = open(self.path, "rb")
return file_stream
class BlobDescriptor:
"""FileDescriptor is used to describe a file that will be used as an ingestion source"""
def __init__(self, path: str, size: Optional[int] = None, source_id: OptionalUUID = None):
"""
:param path: blob uri.
:type path: str.
:param size: estimated size of file if known.
:type size: Optional[int].
:param source_id: a v4 uuid to serve as the sources id.
:type source_id: OptionalUUID
"""
self.path: str = path
self.size: Optional[int] = size
self.source_id: uuid.UUID = ensure_uuid(source_id)
class StreamDescriptor:
"""StreamDescriptor is used to describe a stream that will be used as ingestion source"""
# TODO: currently we always assume that streams are gz compressed (will get compressed before sending), should we expand that?
def __init__(
self, stream: IO[AnyStr], source_id: OptionalUUID = None, is_compressed: bool = False, stream_name: Optional[str] = None, size: Optional[int] = None
):
"""
:param stream: in-memory stream object.
:type stream: io.BaseIO
:param source_id: a v4 uuid to serve as the sources id.
:type source_id: OptionalUUID
:param is_compressed: specify if the provided stream is compressed
:type is_compressed: boolean
"""
self.stream: IO[AnyStr] = stream
self.source_id: uuid.UUID = ensure_uuid(source_id)
self.is_compressed: bool = is_compressed
self.stream_name: str = stream_name
if self.stream_name is None:
self.stream_name = "stream"
if is_compressed:
self.stream_name += ".gz"
self.size: Optional[int] = size
@staticmethod
def from_file_descriptor(file_descriptor: Union[FileDescriptor, str]) -> "StreamDescriptor":
if isinstance(file_descriptor, FileDescriptor):
descriptor = file_descriptor
else:
descriptor = FileDescriptor(file_descriptor)
stream = open(descriptor.path, "rb")
is_compressed = descriptor.path.endswith(".gz") or descriptor.path.endswith(".zip")
stream_descriptor = StreamDescriptor(stream, descriptor.source_id, is_compressed, descriptor.stream_name, descriptor.size)
return stream_descriptor
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/exceptions.py 0000664 0000000 0000000 00000002717 14172227630 0027435 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from azure.kusto.data.data_format import DataFormat, IngestionMappingKind
from azure.kusto.data.exceptions import KustoClientError
class KustoMappingError(KustoClientError):
"""
Raised when the provided mapping arguments are invalid.
"""
class KustoDuplicateMappingError(KustoClientError):
"""
Raised when ingestion properties include both
column mappings and a mapping reference
"""
def __init__(self):
message = "Ingestion properties can't contain both an explicit mapping and a mapping reference."
super(KustoDuplicateMappingError, self).__init__(message)
class KustoMissingMappingError(KustoClientError):
"""
Raised when ingestion properties has data format that requires a mapping and is not defined,
or when provided a mapping kind without a mapping.
"""
class KustoInvalidEndpointError(KustoClientError):
"""Raised when trying to ingest to invalid cluster type."""
def __init__(self, expected_service_type, actual_service_type, suggested_endpoint_url=None):
message = f"You are using '{expected_service_type}' client type, but the provided endpoint is of ServiceType '{actual_service_type}'. Initialize the client with the appropriate endpoint URI"
if suggested_endpoint_url:
message = message + ": '" + suggested_endpoint_url + "'"
super(KustoInvalidEndpointError, self).__init__(message)
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/ingest_client.py 0000664 0000000 0000000 00000020532 14172227630 0030076 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import random
import uuid
from typing import Union, AnyStr, IO, List, Optional
from urllib.parse import urlparse
from azure.storage.blob import BlobServiceClient
from azure.storage.queue import QueueServiceClient, TextBase64EncodePolicy
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
from azure.kusto.data.exceptions import KustoServiceError, KustoBlobError
from .ingestion_blob_info import IngestionBlobInfo
from ._resource_manager import _ResourceManager, _ResourceUri
from .base_ingest_client import BaseIngestClient, IngestionResult, IngestionStatus
from .descriptors import BlobDescriptor, FileDescriptor, StreamDescriptor
from .exceptions import KustoInvalidEndpointError
from .ingestion_properties import IngestionProperties
class QueuedIngestClient(BaseIngestClient):
"""
Queued ingest client provides methods to allow queued ingestion into kusto (ADX).
To learn more about the different types of ingestions and when to use each, visit:
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
"""
_INGEST_PREFIX = "ingest-"
_EXPECTED_SERVICE_TYPE = "DataManagement"
def __init__(self, kcsb: Union[str, KustoConnectionStringBuilder]):
"""Kusto Ingest Client constructor.
:param kcsb: The connection string to initialize KustoClient.
"""
if not isinstance(kcsb, KustoConnectionStringBuilder):
kcsb = KustoConnectionStringBuilder(kcsb)
self._connection_datasource = kcsb.data_source
self._resource_manager = _ResourceManager(KustoClient(kcsb))
self._endpoint_service_type = None
self._suggested_endpoint_uri = None
def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult:
"""Enqueue an ingest command from local files.
To learn more about ingestion methods go to:
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
:param file_descriptor: a FileDescriptor to be ingested.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
containers = self._get_containers()
if isinstance(file_descriptor, FileDescriptor):
descriptor = file_descriptor
else:
descriptor = FileDescriptor(file_descriptor)
should_compress = not descriptor.is_compressed and ingestion_properties.format.compressible
with descriptor.open(should_compress) as stream:
blob_descriptor = QueuedIngestClient._upload_blob(containers, descriptor, ingestion_properties, stream)
result = self.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_properties)
return result
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
"""Ingest from io streams.
:param stream_descriptor: An object that contains a description of the stream to be ingested.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
containers = self._get_containers()
stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties)
blob_descriptor = QueuedIngestClient._upload_blob(containers, stream_descriptor, ingestion_properties, stream_descriptor.stream)
return self.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_properties)
def ingest_from_blob(self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties) -> IngestionResult:
"""Enqueue an ingest command from azure blobs.
To learn more about ingestion methods go to:
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
:param azure.kusto.ingest.BlobDescriptor blob_descriptor: An object that contains a description of the blob to be ingested.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
try:
queues = self._resource_manager.get_ingestion_queues()
except KustoServiceError as ex:
self._validate_endpoint_service_type()
raise ex
random_queue = random.choice(queues)
queue_service = QueueServiceClient(random_queue.account_uri)
authorization_context = self._resource_manager.get_authorization_context()
ingestion_blob_info = IngestionBlobInfo(blob_descriptor, ingestion_properties=ingestion_properties, auth_context=authorization_context)
ingestion_blob_info_json = ingestion_blob_info.to_json()
queue_client = queue_service.get_queue_client(queue=random_queue.object_name, message_encode_policy=TextBase64EncodePolicy())
queue_client.send_message(content=ingestion_blob_info_json)
return IngestionResult(
IngestionStatus.QUEUED, ingestion_properties.database, ingestion_properties.table, blob_descriptor.source_id, blob_descriptor.path
)
def _get_containers(self) -> List[_ResourceUri]:
try:
containers = self._resource_manager.get_containers()
except KustoServiceError as ex:
self._validate_endpoint_service_type()
raise ex
return containers
@staticmethod
def _upload_blob(
containers: List[_ResourceUri],
descriptor: Union[FileDescriptor, StreamDescriptor],
ingestion_properties: IngestionProperties,
stream: IO[AnyStr],
) -> BlobDescriptor:
blob_name = "{db}__{table}__{guid}__{file}".format(
db=ingestion_properties.database, table=ingestion_properties.table, guid=descriptor.source_id, file=descriptor.stream_name
)
random_container = random.choice(containers)
try:
blob_service = BlobServiceClient(random_container.account_uri)
blob_client = blob_service.get_blob_client(container=random_container.object_name, blob=blob_name)
blob_client.upload_blob(data=stream)
except Exception as e:
raise KustoBlobError(e)
return BlobDescriptor(blob_client.url, descriptor.size, descriptor.source_id)
def _validate_endpoint_service_type(self):
if not self._hostname_starts_with_ingest(self._connection_datasource):
if not self._endpoint_service_type:
self._endpoint_service_type = self._retrieve_service_type()
if self._EXPECTED_SERVICE_TYPE != self._endpoint_service_type:
if not self._suggested_endpoint_uri:
self._suggested_endpoint_uri = self._generate_endpoint_suggestion(self._connection_datasource)
if not self._suggested_endpoint_uri:
raise KustoInvalidEndpointError(self._EXPECTED_SERVICE_TYPE, self._endpoint_service_type)
raise KustoInvalidEndpointError(self._EXPECTED_SERVICE_TYPE, self._endpoint_service_type, self._suggested_endpoint_uri)
def _retrieve_service_type(self) -> str:
return self._resource_manager.retrieve_service_type()
def _generate_endpoint_suggestion(self, datasource: str) -> Optional[str]:
"""The default is not passing a suggestion to the exception String"""
endpoint_uri_to_suggest_str = None
if datasource.strip():
try:
endpoint_uri_to_suggest = urlparse(datasource) # Standardize URL formatting
endpoint_uri_to_suggest = urlparse(endpoint_uri_to_suggest.scheme + "://" + self._INGEST_PREFIX + endpoint_uri_to_suggest.hostname)
endpoint_uri_to_suggest_str = endpoint_uri_to_suggest.geturl()
except Exception:
# TODO: Add logging infrastructure so we can tell the user as a warning:
# "Couldn't generate suggested endpoint due to problem parsing datasource, with exception: {ex}. The correct endpoint is usually the Engine endpoint with '{self._INGEST_PREFIX}' prepended to the hostname."
pass
return endpoint_uri_to_suggest_str
def _hostname_starts_with_ingest(self, datasource: str) -> bool:
datasource_uri = urlparse(datasource)
hostname = datasource_uri.hostname
return hostname and hostname.startswith(self._INGEST_PREFIX)
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py 0000664 0000000 0000000 00000006727 14172227630 0031271 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import json
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from azure.kusto.ingest import BlobDescriptor, IngestionProperties
class IngestionBlobInfo:
def __init__(self, blob_descriptor: "BlobDescriptor", ingestion_properties: "IngestionProperties", auth_context=None):
self.properties = dict()
self.properties["BlobPath"] = blob_descriptor.path
if blob_descriptor.size:
self.properties["RawDataSize"] = blob_descriptor.size
self.properties["DatabaseName"] = ingestion_properties.database
self.properties["TableName"] = ingestion_properties.table
self.properties["RetainBlobOnSuccess"] = True
self.properties["FlushImmediately"] = ingestion_properties.flush_immediately
self.properties["IgnoreSizeLimit"] = False
self.properties["ReportLevel"] = ingestion_properties.report_level.value
self.properties["ReportMethod"] = ingestion_properties.report_method.value
self.properties["SourceMessageCreationTime"] = datetime.utcnow().isoformat()
self.properties["Id"] = str(blob_descriptor.source_id)
additional_properties = ingestion_properties.additional_properties or {}
additional_properties["authorizationContext"] = auth_context
tags = []
if ingestion_properties.additional_tags:
tags.extend(ingestion_properties.additional_tags)
if ingestion_properties.drop_by_tags:
tags.extend(["drop-by:" + drop for drop in ingestion_properties.drop_by_tags])
if ingestion_properties.ingest_by_tags:
tags.extend(["ingest-by:" + ingest for ingest in ingestion_properties.ingest_by_tags])
if tags:
additional_properties["tags"] = _convert_list_to_json(tags)
if ingestion_properties.ingest_if_not_exists:
additional_properties["ingestIfNotExists"] = _convert_list_to_json(ingestion_properties.ingest_if_not_exists)
if ingestion_properties.ingestion_mapping:
json_string = _convert_dict_to_json(ingestion_properties.ingestion_mapping)
additional_properties["ingestionMapping"] = json_string
if ingestion_properties.ingestion_mapping_reference:
additional_properties["ingestionMappingReference"] = ingestion_properties.ingestion_mapping_reference
if ingestion_properties.ingestion_mapping_type:
additional_properties["ingestionMappingType"] = ingestion_properties.ingestion_mapping_type.value
if ingestion_properties.validation_policy:
additional_properties["ValidationPolicy"] = _convert_dict_to_json(ingestion_properties.validation_policy)
if ingestion_properties.format:
additional_properties["format"] = ingestion_properties.format.kusto_value
if additional_properties:
self.properties["AdditionalProperties"] = additional_properties
def to_json(self):
"""Converts this object to a json string"""
return _convert_list_to_json(self.properties)
def _convert_list_to_json(array):
"""Converts array to a json string"""
return json.dumps(array, skipkeys=False, allow_nan=False, indent=None, separators=(",", ":"))
def _convert_dict_to_json(array):
"""Converts array to a json string"""
return json.dumps(array, skipkeys=False, allow_nan=False, indent=None, separators=(",", ":"), sort_keys=True, default=lambda o: o.__dict__)
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/ingestion_properties.py 0000664 0000000 0000000 00000016757 14172227630 0031540 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from enum import Enum, IntEnum
from typing import List, Optional
from azure.kusto.data.data_format import DataFormat, IngestionMappingKind
from .exceptions import KustoDuplicateMappingError, KustoMissingMappingError, KustoMappingError
class ValidationOptions(IntEnum):
"""Validation options to ingest command."""
DoNotValidate = 0
ValidateCsvInputConstantColumns = 1
ValidateCsvInputColumnLevelOnly = 2
class ValidationImplications(IntEnum):
"""Validation implications to ingest command."""
Fail = 0
BestEffort = 1
class ValidationPolicy:
"""Validation policy to ingest command."""
def __init__(self, validation_options=ValidationOptions.DoNotValidate, validation_implications=ValidationImplications.BestEffort):
self.ValidationOptions = validation_options
self.ValidationImplications = validation_implications
class ReportLevel(IntEnum):
"""Report level to ingest command."""
FailuresOnly = 0
DoNotReport = 1
FailuresAndSuccesses = 2
class ReportMethod(IntEnum):
"""Report method to ingest command."""
Queue = 0
class TransformationMethod(Enum):
"""Transformations to configure over json column mapping
To read more about mapping transformations look here: https://docs.microsoft.com/en-us/azure/kusto/management/mappings#mapping-transformations"""
NONE = "None"
PROPERTY_BAG_ARRAY_TO_DICTIONARY = ("PropertyBagArrayToDictionary",)
SOURCE_LOCATION = "SourceLocation"
SOURCE_LINE_NUMBER = "SourceLineNumber"
GET_PATH_ELEMENT = "GetPathElement"
UNKNOWN_ERROR = "UnknownMethod"
DATE_TIME_FROM_UNIX_SECONDS = "DateTimeFromUnixSeconds"
DATE_TIME_FROM_UNIX_MILLISECONDS = "DateTimeFromUnixMilliseconds"
DATE_TIME_FROM_UNIX_MICROSECONDS = "DateTimeFromUnixMicroseconds"
DATE_TIME_FROM_UNIX_NANOSECONDS = "DateTimeFromUnixNanoseconds"
class ColumnMapping:
"""Use this class to create mappings for IngestionProperties.ingestionMappings and utilize mappings that were not
pre-created (it is recommended to create the mappings in advance and use ingestionMappingReference).
To read more about mappings look here: https://docs.microsoft.com/en-us/azure/kusto/management/mappings"""
# Json Mapping consts
PATH = "Path"
TRANSFORMATION_METHOD = "Transform"
# csv Mapping consts
ORDINAL = "Ordinal"
CONST_VALUE = "ConstValue"
# Avro Mapping consts
FIELD_NAME = "Field"
COLUMNS = "Columns"
# General Mapping consts
STORAGE_DATA_TYPE = "StorageDataType"
def __init__(
self,
column_name: str,
column_type: str,
path: str = None,
transform: TransformationMethod = TransformationMethod.NONE,
ordinal: int = None,
const_value: str = None,
field=None,
columns=None,
storage_data_type=None,
):
self.column = column_name
self.datatype = column_type
self.properties = {}
if path:
self.properties[self.PATH] = path
if transform != TransformationMethod.NONE:
self.properties[self.TRANSFORMATION_METHOD] = transform.value
if ordinal is not None:
self.properties[self.ORDINAL] = str(ordinal)
if const_value:
self.properties[self.CONST_VALUE] = const_value
if field:
self.properties[self.FIELD_NAME] = field
if columns:
self.properties[self.COLUMNS] = columns
if storage_data_type:
self.properties[self.STORAGE_DATA_TYPE] = storage_data_type
def is_valid(self, kind: IngestionMappingKind) -> bool:
if not self.column:
return False
if kind in (IngestionMappingKind.JSON, IngestionMappingKind.PARQUET, IngestionMappingKind.ORC, IngestionMappingKind.W3CLOGFILE):
return (
bool(self.properties.get(self.PATH))
or self.properties.get(self.TRANSFORMATION_METHOD) == TransformationMethod.SOURCE_LINE_NUMBER.value
or self.properties.get(self.TRANSFORMATION_METHOD) == TransformationMethod.SOURCE_LOCATION.value
)
if kind in (IngestionMappingKind.AVRO, IngestionMappingKind.APACHEAVRO):
return bool(self.properties.get(self.COLUMNS))
return True
class IngestionProperties:
"""
Class to represent ingestion properties.
For more information check out https://docs.microsoft.com/en-us/azure/data-explorer/ingestion-properties
"""
def __init__(
self,
database: str,
table: str,
data_format: DataFormat = DataFormat.CSV,
column_mappings: Optional[List[ColumnMapping]] = None,
ingestion_mapping_kind: Optional[IngestionMappingKind] = None,
ingestion_mapping_reference: Optional[str] = None,
ingest_if_not_exists: Optional[List[str]] = None,
ingest_by_tags: Optional[List[str]] = None,
drop_by_tags: Optional[List[str]] = None,
additional_tags: Optional[List[str]] = None,
flush_immediately: bool = False,
report_level: ReportLevel = ReportLevel.DoNotReport,
report_method: ReportMethod = ReportMethod.Queue,
validation_policy: Optional[ValidationPolicy] = None,
additional_properties: Optional[dict] = None,
):
if ingestion_mapping_reference is None and column_mappings is None:
if data_format._mapping_required:
raise KustoMissingMappingError(f"When stream format is '{data_format.kusto_value}', a mapping must be provided.")
if ingestion_mapping_kind is not None:
raise KustoMissingMappingError(f"When ingestion mapping kind is set ('{ingestion_mapping_kind.value}'), a mapping must be provided.")
else: # A mapping is provided
if ingestion_mapping_kind is not None:
if data_format.ingestion_mapping_kind != ingestion_mapping_kind:
raise KustoMappingError(
f"Wrong ingestion mapping for format '{data_format.kusto_value}'; mapping kind should be '{data_format.ingestion_mapping_kind.value}', "
f"but was '{ingestion_mapping_kind.value}'. "
)
else:
ingestion_mapping_kind = data_format.ingestion_mapping_kind
if column_mappings is not None:
if ingestion_mapping_reference is not None:
raise KustoDuplicateMappingError()
validation_errors = [
f"Column mapping '{mapping.column}' is invalid." for mapping in column_mappings if not mapping.is_valid(ingestion_mapping_kind)
]
if validation_errors:
errors = "\n".join(validation_errors)
raise KustoMappingError(f"Failed with validation errors:\n{errors}")
self.database = database
self.table = table
self.format = data_format
self.ingestion_mapping = column_mappings
self.ingestion_mapping_type = ingestion_mapping_kind
self.ingestion_mapping_reference = ingestion_mapping_reference
self.additional_tags = additional_tags
self.ingest_if_not_exists = ingest_if_not_exists
self.ingest_by_tags = ingest_by_tags
self.drop_by_tags = drop_by_tags
self.flush_immediately = flush_immediately
self.report_level = report_level
self.report_method = report_method
self.validation_policy = validation_policy
self.additional_properties = additional_properties
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/managed_streaming_ingest_client.py 0000664 0000000 0000000 00000013642 14172227630 0033627 0 ustar 00root root 0000000 0000000 import uuid
from io import SEEK_SET
from typing import TYPE_CHECKING, Union, IO, AnyStr
from azure.kusto.data import KustoConnectionStringBuilder
from azure.kusto.data.exceptions import KustoApiError
from . import IngestionProperties, BlobDescriptor, StreamDescriptor, FileDescriptor
from ._retry import ExponentialRetry
from ._stream_extensions import read_until_size_or_end, chain_streams
from .base_ingest_client import BaseIngestClient, IngestionResult
from .ingest_client import QueuedIngestClient
from .streaming_ingest_client import KustoStreamingIngestClient
if TYPE_CHECKING:
pass
class ManagedStreamingIngestClient(BaseIngestClient):
"""
Managed Streaming Ingestion Client.
Will try to ingest with streaming, but if it fails, will fall back to queued ingestion.
Each transient failure will be retried with exponential backoff.
Managed streaming ingest client will fall back to queued if:
- Multiple transient errors were encountered when trying to do streaming ingestion
- The ingestion is too large for streaming ingestion (over 4MB)
- The ingestion is directly for a blob
"""
MAX_STREAMING_SIZE_IN_BYTES = 4 * 1024 * 1024
ATTEMPT_COUNT = 3
@staticmethod
def from_engine_kcsb(engine_kcsb: Union[KustoConnectionStringBuilder, str]) -> "ManagedStreamingIngestClient":
"""
Create a ManagedStreamingIngestClient from a KustoConnectionStringBuilder for the engine.
This Connection String is used for the streaming ingest client.
This method will infer the dm connection string (by using the same authentication and adding the ingest- prefix)
For advanced use cases, use the constructor directly.
:param engine_kcsb: KustoConnectionStringBuilder for the engine.
:return: ManagedStreamingIngestClient
"""
kcsb = repr(engine_kcsb) if type(engine_kcsb) == KustoConnectionStringBuilder else engine_kcsb
dm_kcsb = KustoConnectionStringBuilder(kcsb.replace("https://", "https://ingest-"))
return ManagedStreamingIngestClient(engine_kcsb, dm_kcsb)
@staticmethod
def from_dm_kcsb(dm_kcsb: Union[KustoConnectionStringBuilder, str]) -> "ManagedStreamingIngestClient":
"""
Create a ManagedStreamingIngestClient from a KustoConnectionStringBuilder for the dm.
This Connection String is used for the queued ingest client.
This method will infer the engine connection string (by using the same authentication and removing the ingest- prefix)
For advanced use cases, use the constructor directly.
:param dm_kcsb: KustoConnectionStringBuilder for the dm.
:return: ManagedStreamingIngestClient
"""
kcsb = repr(dm_kcsb) if type(dm_kcsb) == KustoConnectionStringBuilder else dm_kcsb
engine_kcsb = KustoConnectionStringBuilder(kcsb.replace("https://ingest-", "https://"))
return ManagedStreamingIngestClient(engine_kcsb, dm_kcsb)
def __init__(self, engine_kcsb: Union[KustoConnectionStringBuilder, str], dm_kcsb: Union[KustoConnectionStringBuilder, str]):
self.queued_client = QueuedIngestClient(dm_kcsb)
self.streaming_client = KustoStreamingIngestClient(engine_kcsb)
def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult:
stream_descriptor = StreamDescriptor.from_file_descriptor(file_descriptor)
with stream_descriptor.stream:
return self.ingest_from_stream(stream_descriptor, ingestion_properties)
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties)
stream = stream_descriptor.stream
buffered_stream = read_until_size_or_end(stream, self.MAX_STREAMING_SIZE_IN_BYTES + 1)
if len(buffered_stream.getbuffer()) > self.MAX_STREAMING_SIZE_IN_BYTES:
stream_descriptor.stream = chain_streams([buffered_stream, stream])
return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties)
stream_descriptor.stream = buffered_stream
retry = self._create_exponential_retry()
while retry:
try:
client_request_id = ManagedStreamingIngestClient._get_request_id(stream_descriptor.source_id, retry.current_attempt)
return self.streaming_client._ingest_from_stream_with_client_request_id(stream_descriptor, ingestion_properties, client_request_id)
except KustoApiError as e:
error = e.get_api_error()
if error.permanent:
raise
stream.seek(0, SEEK_SET)
retry.do_backoff()
return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties)
def ingest_from_blob(self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties):
"""
Enqueue an ingest command from azure blobs.
For ManagedStreamingIngestClient, this method always uses Queued Ingest, since it would be easier and faster to ingest blobs.
To learn more about ingestion methods go to:
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
:param azure.kusto.ingest.BlobDescriptor blob_descriptor: An object that contains a description of the blob to be ingested.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
return self.queued_client.ingest_from_blob(blob_descriptor, ingestion_properties)
@staticmethod
def _get_request_id(source_id: uuid.UUID, attempt: int):
return f"KPC.executeManagedStreamingIngest;{source_id};{attempt}"
@staticmethod
def _create_exponential_retry():
return ExponentialRetry(ManagedStreamingIngestClient.ATTEMPT_COUNT)
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/status.py 0000664 0000000 0000000 00000003022 14172227630 0026565 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import json
from ._status_q import StatusQueue
class StatusMessage:
OperationId = None
Database = None
Table = None
IngestionSourceId = None
IngestionSourcePath = None
RootActivityId = None
_raw = None
def __init__(self, s):
self._raw = s
o = json.loads(s)
for key, value in o.items():
if hasattr(self, key):
try:
setattr(self, key, value)
except:
# TODO: should we set up a logger?
pass
def __str__(self):
return "{}".format(self._raw)
def __repr__(self):
return "{0.__class__.__name__}({0._raw})".format(self)
class SuccessMessage(StatusMessage):
SucceededOn = None
class FailureMessage(StatusMessage):
FailedOn = None
Details = None
ErrorCode = None
FailureStatus = None
OriginatesFromUpdatePolicy = None
ShouldRetry = None
class KustoIngestStatusQueues:
"""Kusto ingest Status Queue.
Use this class to get status messages from Kusto status queues.
Currently there are two queues exposed: `failure` and `success` queues.
"""
def __init__(self, kusto_ingest_client):
self.success = StatusQueue(kusto_ingest_client._resource_manager.get_successful_ingestions_queues, message_cls=SuccessMessage)
self.failure = StatusQueue(kusto_ingest_client._resource_manager.get_failed_ingestions_queues, message_cls=FailureMessage)
azure-kusto-python-3.0.1/azure-kusto-ingest/azure/kusto/ingest/streaming_ingest_client.py 0000664 0000000 0000000 00000006045 14172227630 0032152 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from typing import Union, AnyStr, Optional
from typing import IO
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties
from .base_ingest_client import BaseIngestClient, IngestionResult, IngestionStatus
from .descriptors import FileDescriptor, StreamDescriptor
from .ingestion_properties import IngestionProperties
class KustoStreamingIngestClient(BaseIngestClient):
"""Kusto streaming ingest client for Python.
KustoStreamingIngestClient works with both 2.x and 3.x flavors of Python.
All primitive types are supported.
Tests are run using pytest.
"""
def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str]):
"""Kusto Streaming Ingest Client constructor.
:param KustoConnectionStringBuilder kcsb: The connection string to initialize KustoClient.
"""
self._kusto_client = KustoClient(kcsb)
def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult:
"""Ingest from local files.
:param file_descriptor: a FileDescriptor to be ingested.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
stream_descriptor = StreamDescriptor.from_file_descriptor(file_descriptor)
with stream_descriptor.stream:
return self.ingest_from_stream(stream_descriptor, ingestion_properties)
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
"""Ingest from io streams.
:param azure.kusto.ingest.StreamDescriptor stream_descriptor: An object that contains a description of the stream to
be ingested.
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
"""
return self._ingest_from_stream_with_client_request_id(stream_descriptor, ingestion_properties, None)
def _ingest_from_stream_with_client_request_id(
self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties, client_request_id: Optional[str]
) -> IngestionResult:
stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties)
additional_properties = None
if client_request_id:
additional_properties = ClientRequestProperties()
additional_properties.client_request_id = client_request_id
self._kusto_client.execute_streaming_ingest(
ingestion_properties.database,
ingestion_properties.table,
stream_descriptor.stream,
ingestion_properties.format.name,
additional_properties,
mapping_name=ingestion_properties.ingestion_mapping_reference,
)
return IngestionResult(IngestionStatus.SUCCESS, ingestion_properties.database, ingestion_properties.table, stream_descriptor.source_id)
azure-kusto-python-3.0.1/azure-kusto-ingest/setup.cfg 0000664 0000000 0000000 00000000227 14172227630 0022731 0 ustar 00root root 0000000 0000000 [bdist_wheel]
universal=1
[flake8]
ignore = E226,E302,E41
max-line-length = 160
exclude = tests/*
max-complexity = 10
[pylint]
max-line-length = 160
azure-kusto-python-3.0.1/azure-kusto-ingest/setup.py 0000664 0000000 0000000 00000003033 14172227630 0022620 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import re
from os import path
from setuptools import setup, find_packages
PACKAGE_NAME = "azure-kusto-ingest"
# a-b-c => a/b/c
PACKAGE_FOLDER_PATH = PACKAGE_NAME.replace("-", path.sep)
# a-b-c => a.b.c
NAMESPACE_NAME = PACKAGE_NAME.replace("-", ".")
with open(path.join(PACKAGE_FOLDER_PATH, "_version.py"), "r") as fd:
VERSION = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1)
if not VERSION:
raise RuntimeError("Cannot find version information")
setup(
name=PACKAGE_NAME,
version=VERSION,
description="Kusto Ingest Client",
long_description=open("README.rst", "r").read(),
license="MIT",
author="Microsoft Corporation",
author_email="kustalk@microsoft.com",
url="https://github.com/Azure/azure-kusto-python",
namespace_packages=["azure"],
classifiers=[
# 5 - Production/Stable depends on multi-threading / aio / perf
"Development Status :: 4 - Beta",
"Programming Language :: Python",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"License :: OSI Approved :: MIT License",
],
packages=find_packages(exclude=["azure", "tests"]),
install_requires=["azure-kusto-data=={}".format(VERSION), "azure-storage-blob>=12,<13", "azure-storage-queue>=12,<13"],
extras_require={"pandas": ["pandas"], "aio": []},
)
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/ 0000775 0000000 0000000 00000000000 14172227630 0022251 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-ingest/tests/e2e.py 0000664 0000000 0000000 00000071433 14172227630 0023306 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import io
import json
import os
import pathlib
import random
import sys
import time
import unittest
import uuid
from datetime import datetime
from typing import Optional
import pytest
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
from azure.kusto.data._cloud_settings import CloudSettings
from azure.kusto.data._models import WellKnownDataSet
from azure.kusto.data.aio import KustoClient as AsyncKustoClient
from azure.kusto.data.data_format import DataFormat, IngestionMappingKind
from azure.kusto.data.exceptions import KustoServiceError
from azure.kusto.data.streaming_response import FrameType
from azure.kusto.ingest import (
QueuedIngestClient,
KustoStreamingIngestClient,
IngestionProperties,
ColumnMapping,
ValidationPolicy,
ValidationOptions,
ValidationImplications,
ReportLevel,
ReportMethod,
FileDescriptor,
BlobDescriptor,
StreamDescriptor,
ManagedStreamingIngestClient,
)
@pytest.fixture(params=["ManagedStreaming", "NormalClient"])
def is_managed_streaming(request):
return request.param == "ManagedStreaming"
class TestE2E:
"""A class to define mappings to deft table."""
input_folder_path: str
streaming_test_table: str
test_streaming_data: list
engine_cs: Optional[str]
dm_cs: Optional[str]
app_id: Optional[str]
app_key: Optional[str]
auth_id: Optional[str]
test_db: Optional[str]
client: KustoClient
test_table: str
current_count: int
CHUNK_SIZE = 1024
@staticmethod
def get_test_table_csv_mappings():
"""A method to define csv mappings to test table."""
mappings = list()
mappings.append(ColumnMapping(column_name="rownumber", column_type="int", ordinal=0))
mappings.append(ColumnMapping(column_name="rowguid", column_type="string", ordinal=1))
mappings.append(ColumnMapping(column_name="xdouble", column_type="real", ordinal=2))
mappings.append(ColumnMapping(column_name="xfloat", column_type="real", ordinal=3))
mappings.append(ColumnMapping(column_name="xbool", column_type="bool", ordinal=4))
mappings.append(ColumnMapping(column_name="xint16", column_type="int", ordinal=5))
mappings.append(ColumnMapping(column_name="xint32", column_type="int", ordinal=6))
mappings.append(ColumnMapping(column_name="xint64", column_type="long", ordinal=7))
mappings.append(ColumnMapping(column_name="xuint8", column_type="long", ordinal=8))
mappings.append(ColumnMapping(column_name="xuint16", column_type="long", ordinal=9))
mappings.append(ColumnMapping(column_name="xuint32", column_type="long", ordinal=10))
mappings.append(ColumnMapping(column_name="xuint64", column_type="long", ordinal=11))
mappings.append(ColumnMapping(column_name="xdate", column_type="datetime", ordinal=12))
mappings.append(ColumnMapping(column_name="xsmalltext", column_type="string", ordinal=13))
mappings.append(ColumnMapping(column_name="xtext", column_type="string", ordinal=14))
mappings.append(ColumnMapping(column_name="xnumberAsText", column_type="string", ordinal=15))
mappings.append(ColumnMapping(column_name="xtime", column_type="timespan", ordinal=16))
mappings.append(ColumnMapping(column_name="xtextWithNulls", column_type="string", ordinal=17))
mappings.append(ColumnMapping(column_name="xdynamicWithNulls", column_type="dynamic", ordinal=18))
return mappings
@staticmethod
def test_table_json_mappings():
"""A method to define json mappings to test table."""
mappings = list()
mappings.append(ColumnMapping(column_name="rownumber", path="$.rownumber", column_type="int"))
mappings.append(ColumnMapping(column_name="rowguid", path="$.rowguid", column_type="string"))
mappings.append(ColumnMapping(column_name="xdouble", path="$.xdouble", column_type="real"))
mappings.append(ColumnMapping(column_name="xfloat", path="$.xfloat", column_type="real"))
mappings.append(ColumnMapping(column_name="xbool", path="$.xbool", column_type="bool"))
mappings.append(ColumnMapping(column_name="xint16", path="$.xint16", column_type="int"))
mappings.append(ColumnMapping(column_name="xint32", path="$.xint32", column_type="int"))
mappings.append(ColumnMapping(column_name="xint64", path="$.xint64", column_type="long"))
mappings.append(ColumnMapping(column_name="xuint8", path="$.xuint8", column_type="long"))
mappings.append(ColumnMapping(column_name="xuint16", path="$.xuint16", column_type="long"))
mappings.append(ColumnMapping(column_name="xuint32", path="$.xuint32", column_type="long"))
mappings.append(ColumnMapping(column_name="xuint64", path="$.xuint64", column_type="long"))
mappings.append(ColumnMapping(column_name="xdate", path="$.xdate", column_type="datetime"))
mappings.append(ColumnMapping(column_name="xsmalltext", path="$.xsmalltext", column_type="string"))
mappings.append(ColumnMapping(column_name="xtext", path="$.xtext", column_type="string"))
mappings.append(ColumnMapping(column_name="xnumberAsText", path="$.xnumberAsText", column_type="string"))
mappings.append(ColumnMapping(column_name="xtime", path="$.xtime", column_type="timespan"))
mappings.append(ColumnMapping(column_name="xtextWithNulls", path="$.xtextWithNulls", column_type="string"))
mappings.append(ColumnMapping(column_name="xdynamicWithNulls", path="$.xdynamicWithNulls", column_type="dynamic"))
return mappings
@staticmethod
def test_table_json_mapping_reference():
"""A method to get json mappings reference to test table."""
return """'['
' { "column" : "rownumber", "datatype" : "int", "Properties":{"Path":"$.rownumber"}},'
' { "column" : "rowguid", "datatype" : "string", "Properties":{"Path":"$.rowguid"}},'
' { "column" : "xdouble", "datatype" : "real", "Properties":{"Path":"$.xdouble"}},'
' { "column" : "xfloat", "datatype" : "real", "Properties":{"Path":"$.xfloat"}},'
' { "column" : "xbool", "datatype" : "bool", "Properties":{"Path":"$.xbool"}},'
' { "column" : "xint16", "datatype" : "int", "Properties":{"Path":"$.xint16"}},'
' { "column" : "xint32", "datatype" : "int", "Properties":{"Path":"$.xint32"}},'
' { "column" : "xint64", "datatype" : "long", "Properties":{"Path":"$.xint64"}},'
' { "column" : "xuint8", "datatype" : "long", "Properties":{"Path":"$.xuint8"}},'
' { "column" : "xuint16", "datatype" : "long", "Properties":{"Path":"$.xuint16"}},'
' { "column" : "xuint32", "datatype" : "long", "Properties":{"Path":"$.xuint32"}},'
' { "column" : "xuint64", "datatype" : "long", "Properties":{"Path":"$.xuint64"}},'
' { "column" : "xdate", "datatype" : "datetime", "Properties":{"Path":"$.xdate"}},'
' { "column" : "xsmalltext", "datatype" : "string", "Properties":{"Path":"$.xsmalltext"}},'
' { "column" : "xtext", "datatype" : "string", "Properties":{"Path":"$.xtext"}},'
' { "column" : "xnumberAsText", "datatype" : "string", "Properties":{"Path":"$.rowguid"}},'
' { "column" : "xtime", "datatype" : "timespan", "Properties":{"Path":"$.xtime"}},'
' { "column" : "xtextWithNulls", "datatype" : "string", "Properties":{"Path":"$.xtextWithNulls"}},'
' { "column" : "xdynamicWithNulls", "datatype" : "dynamic", "Properties":{"Path":"$.xdynamicWithNulls"}},'
']'"""
@staticmethod
def get_file_path() -> str:
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
return os.path.join(current_dir, *missing_path_parts)
@classmethod
def engine_kcsb_from_env(cls) -> KustoConnectionStringBuilder:
if all([cls.app_id, cls.app_key, cls.auth_id]):
return KustoConnectionStringBuilder.with_aad_application_key_authentication(cls.engine_cs, cls.app_id, cls.app_key, cls.auth_id)
else:
return KustoConnectionStringBuilder.with_interactive_login(cls.engine_cs)
@classmethod
def dm_kcsb_from_env(cls) -> KustoConnectionStringBuilder:
if all([cls.app_id, cls.app_key, cls.auth_id]):
return KustoConnectionStringBuilder.with_aad_application_key_authentication(cls.dm_cs, cls.app_id, cls.app_key, cls.auth_id)
else:
return KustoConnectionStringBuilder.with_interactive_login(cls.dm_cs)
@classmethod
def setup_class(cls):
# DM CS can be composed from engine CS
cls.engine_cs = os.environ.get("ENGINE_CONNECTION_STRING")
cls.dm_cs = os.environ.get("DM_CONNECTION_STRING") or cls.engine_cs.replace("//", "//ingest-")
cls.app_id = os.environ.get("APP_ID")
cls.app_key = os.environ.get("APP_KEY")
cls.auth_id = os.environ.get("AUTH_ID")
cls.test_db = os.environ.get("TEST_DATABASE")
cls.test_blob = os.environ.get("TEST_BLOB")
if not all([cls.engine_cs, cls.dm_cs, cls.test_db]):
raise unittest.SkipTest("E2E environment is missing")
# Init clients
python_version = "_".join([str(v) for v in sys.version_info[:3]])
cls.test_table = "python_test_{0}_{1}_{2}".format(python_version, str(int(time.time())), random.randint(1, 100000))
cls.streaming_test_table = "BigChunkus"
cls.streaming_test_table_query = cls.streaming_test_table + " | order by timestamp"
cls.client = KustoClient(cls.engine_kcsb_from_env())
cls.ingest_client = QueuedIngestClient(cls.dm_kcsb_from_env())
cls.streaming_ingest_client = KustoStreamingIngestClient(cls.engine_kcsb_from_env())
cls.managed_streaming_ingest_client = ManagedStreamingIngestClient(cls.engine_kcsb_from_env(), cls.dm_kcsb_from_env())
cls.input_folder_path = cls.get_file_path()
cls.csv_file_path = os.path.join(cls.input_folder_path, "dataset.csv")
cls.tsv_file_path = os.path.join(cls.input_folder_path, "dataset.tsv")
cls.zipped_csv_file_path = os.path.join(cls.input_folder_path, "dataset.csv.gz")
cls.json_file_path = os.path.join(cls.input_folder_path, "dataset.json")
cls.zipped_json_file_path = os.path.join(cls.input_folder_path, "dataset.jsonz.gz")
with open(os.path.join(cls.input_folder_path, "big.json")) as f:
cls.test_streaming_data = json.load(f)
cls.current_count = 0
cls.client.execute(
cls.test_db,
f".create table {cls.test_table} (rownumber: int, rowguid: string, xdouble: real, xfloat: real, xbool: bool, xint16: int, xint32: int, xint64: long, xuint8: long, xuint16: long, xuint32: long, xuint64: long, xdate: datetime, xsmalltext: string, xtext: string, xnumberAsText: string, xtime: timespan, xtextWithNulls: string, xdynamicWithNulls: dynamic)",
)
cls.client.execute(cls.test_db, f".create table {cls.test_table} ingestion json mapping 'JsonMapping' {cls.test_table_json_mapping_reference()}")
cls.client.execute(cls.test_db, f".alter table {cls.test_table} policy streamingingestion enable ")
# Clear the cache to guarantee that subsequent streaming ingestion requests incorporate database and table schema changes
# See https://docs.microsoft.com/azure/data-explorer/kusto/management/data-ingestion/clear-schema-cache-command
cls.client.execute(cls.test_db, ".clear database cache streamingingestion schema")
@classmethod
def teardown_class(cls):
cls.client.execute(cls.test_db, ".drop table {} ifexists".format(cls.test_table))
@classmethod
async def get_async_client(cls) -> AsyncKustoClient:
return AsyncKustoClient(cls.engine_kcsb_from_env())
# assertions
@classmethod
async def assert_rows_added(cls, expected: int, timeout=60):
actual = 0
while timeout > 0:
time.sleep(1)
timeout -= 1
try:
command = "{} | count".format(cls.test_table)
response = cls.client.execute(cls.test_db, command)
async_client = await cls.get_async_client()
response_from_async = await async_client.execute(cls.test_db, command)
except KustoServiceError:
continue
if response is not None:
row = response.primary_results[0][0]
row_async = response_from_async.primary_results[0][0]
actual = int(row["Count"]) - cls.current_count
# this is done to allow for data to arrive properly
if actual >= expected:
assert row_async == row, "Mismatch answers between async('{0}') and sync('{1}') clients".format(row_async, row)
break
cls.current_count += actual
assert actual == expected, "Row count expected = {0}, while actual row count = {1}".format(expected, actual)
@staticmethod
def normalize_row(row):
result = []
for r in row:
if type(r) == bool:
result.append(int(r))
elif type(r) == datetime:
result.append(r.strftime("%Y-%m-%dT%H:%M:%SZ"))
else:
result.append(r)
return result
def test_streaming_query(self):
result = self.client.execute_streaming_query(self.test_db, self.streaming_test_table_query + ";" + self.streaming_test_table_query)
counter = 0
result.set_skip_incomplete_tables(True)
for primary in result.iter_primary_results():
counter += 1
for row in self.test_streaming_data:
assert row == self.normalize_row(next(primary).to_list())
assert counter == 2
assert result.finished
assert result.errors_count == 0
assert result.get_exceptions() == []
@pytest.mark.asyncio
async def test_streaming_query_async(self):
async with await self.get_async_client() as client:
result = await client.execute_streaming_query(self.test_db, self.streaming_test_table_query + ";" + self.streaming_test_table_query)
counter = 0
result.set_skip_incomplete_tables(True)
async for primary in result.iter_primary_results():
counter += 1
streaming_data_iter = iter(self.test_streaming_data)
async for row in primary:
expected_row = next(streaming_data_iter, None)
if expected_row is None:
break
assert expected_row == self.normalize_row(row.to_list())
assert counter == 2
assert result.finished
assert result.errors_count == 0
assert result.get_exceptions() == []
def test_streaming_query_internal(self):
frames = self.client._execute_streaming_query_parsed(self.test_db, self.streaming_test_table_query)
initial_frame = next(frames)
expected_initial_frame = {
"FrameType": FrameType.DataSetHeader,
"IsProgressive": False,
"Version": "v2.0",
}
assert initial_frame == expected_initial_frame
query_props = next(frames)
assert query_props["FrameType"] == FrameType.DataTable
assert query_props["TableKind"] == WellKnownDataSet.QueryProperties.value
assert type(query_props["Columns"]) == list
assert type(query_props["Rows"]) == list
assert len(query_props["Rows"][0]) == len(query_props["Columns"])
primary_result = next(frames)
assert primary_result["FrameType"] == FrameType.DataTable
assert primary_result["TableKind"] == WellKnownDataSet.PrimaryResult.value
assert type(primary_result["Columns"]) == list
assert type(primary_result["Rows"]) != list
row = next(primary_result["Rows"])
assert len(row) == len(primary_result["Columns"])
@pytest.mark.asyncio
async def test_streaming_query_internal_async(self):
async with await self.get_async_client() as client:
frames = await client._execute_streaming_query_parsed(self.test_db, self.streaming_test_table_query)
frames.__aiter__()
initial_frame = await frames.__anext__()
expected_initial_frame = {
"FrameType": FrameType.DataSetHeader,
"IsProgressive": False,
"Version": "v2.0",
}
assert initial_frame == expected_initial_frame
query_props = await frames.__anext__()
assert query_props["FrameType"] == FrameType.DataTable
assert query_props["TableKind"] == WellKnownDataSet.QueryProperties.value
assert type(query_props["Columns"]) == list
assert type(query_props["Rows"]) == list
assert len(query_props["Rows"][0]) == len(query_props["Columns"])
primary_result = await frames.__anext__()
assert primary_result["FrameType"] == FrameType.DataTable
assert primary_result["TableKind"] == WellKnownDataSet.PrimaryResult.value
assert type(primary_result["Columns"]) == list
assert type(primary_result["Rows"]) != list
row = await primary_result["Rows"].__anext__()
assert len(row) == len(primary_result["Columns"])
@pytest.mark.asyncio
async def test_csv_ingest_existing_table(self, is_managed_streaming):
csv_ingest_props = IngestionProperties(
self.test_db,
self.test_table,
data_format=DataFormat.CSV,
column_mappings=self.get_test_table_csv_mappings(),
report_level=ReportLevel.FailuresAndSuccesses,
flush_immediately=True,
)
client = self.streaming_ingest_client if is_managed_streaming else self.ingest_client
for f in [self.csv_file_path, self.zipped_csv_file_path]:
client.ingest_from_file(f, csv_ingest_props)
await self.assert_rows_added(20)
@pytest.mark.asyncio
async def test_json_ingest_existing_table(self):
json_ingestion_props = IngestionProperties(
self.test_db,
self.test_table,
flush_immediately=True,
data_format=DataFormat.JSON,
column_mappings=self.test_table_json_mappings(),
report_level=ReportLevel.FailuresAndSuccesses,
)
for f in [self.json_file_path, self.zipped_json_file_path]:
self.ingest_client.ingest_from_file(f, json_ingestion_props)
await self.assert_rows_added(4)
@pytest.mark.asyncio
async def test_ingest_complicated_props(self):
validation_policy = ValidationPolicy(
validation_options=ValidationOptions.ValidateCsvInputConstantColumns, validation_implications=ValidationImplications.Fail
)
json_ingestion_props = IngestionProperties(
self.test_db,
self.test_table,
data_format=DataFormat.JSON,
column_mappings=self.test_table_json_mappings(),
additional_tags=["a", "b"],
ingest_if_not_exists=["aaaa", "bbbb"],
ingest_by_tags=["ingestByTag"],
drop_by_tags=["drop", "drop-by"],
flush_immediately=False,
report_level=ReportLevel.FailuresAndSuccesses,
report_method=ReportMethod.Queue,
validation_policy=validation_policy,
)
file_paths = [self.json_file_path, self.zipped_json_file_path]
fds = [FileDescriptor(fp, 0, uuid.uuid4()) for fp in file_paths]
for fd in fds:
self.ingest_client.ingest_from_file(fd, json_ingestion_props)
await self.assert_rows_added(4)
@pytest.mark.asyncio
async def test_ingest_from_stream(self, is_managed_streaming):
validation_policy = ValidationPolicy(
validation_options=ValidationOptions.ValidateCsvInputConstantColumns, validation_implications=ValidationImplications.Fail
)
json_ingestion_props = IngestionProperties(
self.test_db,
self.test_table,
data_format=DataFormat.JSON,
column_mappings=self.test_table_json_mappings(),
additional_tags=["a", "b"],
ingest_if_not_exists=["aaaa", "bbbb"],
ingest_by_tags=["ingestByTag"],
drop_by_tags=["drop", "drop-by"],
flush_immediately=False,
report_level=ReportLevel.FailuresAndSuccesses,
report_method=ReportMethod.Queue,
validation_policy=validation_policy,
)
text = io.StringIO(pathlib.Path(self.json_file_path).read_text())
zipped = io.BytesIO(pathlib.Path(self.zipped_json_file_path).read_bytes())
client = self.managed_streaming_ingest_client if is_managed_streaming else self.ingest_client
client.ingest_from_stream(text, json_ingestion_props)
client.ingest_from_stream(StreamDescriptor(zipped, is_compressed=True), json_ingestion_props)
await self.assert_rows_added(4)
@pytest.mark.asyncio
async def test_json_ingestion_ingest_by_tag(self):
json_ingestion_props = IngestionProperties(
self.test_db,
self.test_table,
data_format=DataFormat.JSON,
column_mappings=self.test_table_json_mappings(),
ingest_if_not_exists=["ingestByTag"],
report_level=ReportLevel.FailuresAndSuccesses,
drop_by_tags=["drop", "drop-by"],
flush_immediately=True,
)
for f in [self.json_file_path, self.zipped_json_file_path]:
self.ingest_client.ingest_from_file(f, json_ingestion_props)
await self.assert_rows_added(0)
@pytest.mark.asyncio
async def test_tsv_ingestion_csv_mapping(self):
tsv_ingestion_props = IngestionProperties(
self.test_db,
self.test_table,
flush_immediately=True,
data_format=DataFormat.TSV,
column_mappings=self.get_test_table_csv_mappings(),
report_level=ReportLevel.FailuresAndSuccesses,
)
self.ingest_client.ingest_from_file(self.tsv_file_path, tsv_ingestion_props)
await self.assert_rows_added(10)
@pytest.mark.asyncio
async def test_ingest_blob(self):
if not self.test_blob:
pytest.skip("Provide blob SAS uri with 'dataset.csv'")
csv_ingest_props = IngestionProperties(
self.test_db,
self.test_table,
data_format=DataFormat.CSV,
column_mappings=self.get_test_table_csv_mappings(),
report_level=ReportLevel.FailuresAndSuccesses,
flush_immediately=True,
)
blob_len = 1578
self.ingest_client.ingest_from_blob(BlobDescriptor(self.test_blob, blob_len), csv_ingest_props)
await self.assert_rows_added(10)
# Don't provide size hint
self.ingest_client.ingest_from_blob(BlobDescriptor(self.test_blob, size=None), csv_ingest_props)
await self.assert_rows_added(10)
@pytest.mark.asyncio
async def test_streaming_ingest_from_opened_file(self, is_managed_streaming):
ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, data_format=DataFormat.CSV)
client = self.managed_streaming_ingest_client if is_managed_streaming else self.streaming_ingest_client
with open(self.csv_file_path, "r") as stream:
client.ingest_from_stream(stream, ingestion_properties=ingestion_properties)
await self.assert_rows_added(10, timeout=120)
@pytest.mark.asyncio
async def test_streaming_ingest_from_csv_file(self):
ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, flush_immediately=True, data_format=DataFormat.CSV)
for f in [self.csv_file_path, self.zipped_csv_file_path]:
self.streaming_ingest_client.ingest_from_file(f, ingestion_properties=ingestion_properties)
await self.assert_rows_added(20, timeout=120)
@pytest.mark.asyncio
async def test_streaming_ingest_from_json_file(self):
ingestion_properties = IngestionProperties(
database=self.test_db,
table=self.test_table,
flush_immediately=True,
data_format=DataFormat.JSON,
ingestion_mapping_reference="JsonMapping",
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
for f in [self.json_file_path, self.zipped_json_file_path]:
self.streaming_ingest_client.ingest_from_file(f, ingestion_properties=ingestion_properties)
await self.assert_rows_added(4, timeout=120)
@pytest.mark.asyncio
async def test_streaming_ingest_from_csv_io_streams(self):
ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, data_format=DataFormat.CSV)
byte_sequence = b'0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null'
bytes_stream = io.BytesIO(byte_sequence)
self.streaming_ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
str_sequence = '0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null'
str_stream = io.StringIO(str_sequence)
self.streaming_ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
await self.assert_rows_added(2, timeout=120)
@pytest.mark.asyncio
async def test_streaming_ingest_from_json_io_streams(self):
ingestion_properties = IngestionProperties(
database=self.test_db,
table=self.test_table,
data_format=DataFormat.JSON,
flush_immediately=True,
ingestion_mapping_reference="JsonMapping",
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
byte_sequence = b'{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}'
bytes_stream = io.BytesIO(byte_sequence)
self.streaming_ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
str_sequence = '{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}'
str_stream = io.StringIO(str_sequence)
self.streaming_ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
await self.assert_rows_added(2, timeout=120)
@pytest.mark.asyncio
async def test_streaming_ingest_from_dataframe(self):
from pandas import DataFrame
fields = [
"rownumber",
"rowguid",
"xdouble",
"xfloat",
"xbool",
"xint16",
"xint32",
"xint64",
"xunit8",
"xuint16",
"xunit32",
"xunit64",
"xdate",
"xsmalltext",
"xtext",
"xnumberAsText",
"xtime",
"xtextWithNulls",
"xdynamicWithNulls",
]
rows = [
[0, "00000000-0000-0000-0001-020304050607", 0.0, 0.0, 0, 0, 0, 0, 0, 0, 0, 0, "2014-01-01T01:01:01Z", "Zero", "Zero", "0", "00:00:00", None, ""]
]
df = DataFrame(data=rows, columns=fields)
ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, flush_immediately=True, data_format=DataFormat.CSV)
self.ingest_client.ingest_from_dataframe(df, ingestion_properties)
await self.assert_rows_added(1, timeout=120)
def test_cloud_info(self):
cloud_info = CloudSettings.get_cloud_info_for_cluster(self.engine_cs)
assert cloud_info is not CloudSettings.DEFAULT_CLOUD
assert cloud_info == CloudSettings.DEFAULT_CLOUD
assert cloud_info is CloudSettings.get_cloud_info_for_cluster(self.engine_cs)
def test_cloud_info_404(self):
cloud_info = CloudSettings.get_cloud_info_for_cluster("https://www.microsoft.com")
assert cloud_info is CloudSettings.DEFAULT_CLOUD
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/ 0000775 0000000 0000000 00000000000 14172227630 0023410 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/__init__.py 0000664 0000000 0000000 00000000000 14172227630 0025507 0 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/big.json 0000664 0000000 0000000 00000244446 14172227630 0025062 0 ustar 00root root 0000000 0000000 [
[
952.45224445872259,
0,
"Kibana Airlines",
"Shanghai Hongqiao International Airport",
"SHA",
"Shanghai",
"CN",
{
"lat": "31.19790077",
"lon": "121.3359985"
},
"SE-BD",
"Rain",
9243.8109634707889,
5743.8378391883825,
0,
0,
"No Delay",
"GDZWNB0",
12.838626338153874,
770.31758028923241,
"London Gatwick Airport",
"LGW",
"London",
"GB",
{
"lat": "51.14810181",
"lon": "-0.190277994"
},
"GB-ENG",
"Clear",
6,
"2018-02-11T23:50:12Z"
],
[
1028.7682366113208,
0,
"Logstash Airways",
"Tokyo Haneda International Airport",
"HND",
"Tokyo",
"JP",
{
"lat": "35.552299",
"lon": "139.779999"
},
"SE-BD",
"Rain",
7017.3466765227058,
4360.3770707336062,
0,
0,
"No Delay",
"OKZ5QTE",
5.8477888971022542,
350.86733382613528,
"Olenya Air Base",
"XLMO",
"Olenegorsk",
"RU",
{
"lat": "68.15180206",
"lon": "33.46390152"
},
"RU-MUR",
"Clear",
6,
"2018-02-11T23:46:01Z"
],
[
1091.8802959347431,
0,
"JetBeats",
"King Shaka International Airport",
"DUR",
"Durban",
"ZA",
{
"lat": "-29.61444444",
"lon": "31.11972222"
},
"SE-BD",
"Clear",
8504.6098431600785,
5284.5195577577433,
0,
0,
"No Delay",
"RM652PY",
11.81195811550011,
708.71748693000654,
"Genoa Cristoforo Colombo Airport",
"GE01",
"Genova",
"IT",
{
"lat": "44.4133",
"lon": "8.8375"
},
"IT-42",
"Thunder & Lightning",
6,
"2018-02-11T23:29:45Z"
],
[
440.32475229026517,
1,
"Kibana Airlines",
"Seattle Tacoma International Airport",
"SEA",
"Seattle",
"US",
{
"lat": "47.44900131",
"lon": "-122.3089981"
},
"US-WA",
"Thunder & Lightning",
3419.5367460130756,
2124.8016247695182,
0,
0,
"No Delay",
"KUO775Y",
5.1811162818379932,
310.86697691027962,
"Pittsburgh International Airport",
"PIT",
"Pittsburgh",
"US",
{
"lat": "40.49150085",
"lon": "-80.23290253"
},
"US-PA",
"Clear",
6,
"2018-02-11T23:27:00Z"
],
[
908.84073603059437,
0,
"JetBeats",
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Sunny",
6804.6889820233691,
4228.2377055641109,
0,
0,
"No Delay",
"SE3S0ZP",
7.0882176896076761,
425.29306137646057,
"Rajiv Gandhi International Airport",
"HYD",
"Hyderabad",
"IN",
{
"lat": "17.23131752",
"lon": "78.42985535"
},
"SE-BD",
"Heavy Fog",
6,
"2018-02-11T23:21:27Z"
],
[
760.33982094876387,
1,
"ES-Air",
"Vienna International Airport",
"VIE",
"Vienna",
"AT",
{
"lat": "48.11029816",
"lon": "16.56970024"
},
"AT-9",
"Cloudy",
657.92022778356056,
408.81267633492934,
0,
0,
"No Delay",
"MY1XRHF",
0.6091853960958894,
36.551123765753367,
"Malpensa International Airport",
"MI12",
"Milan",
"IT",
{
"lat": "45.6306",
"lon": "8.72811"
},
"IT-25",
"Thunder & Lightning",
6,
"2018-02-11T23:12:33Z"
],
[
1000.6070389356696,
0,
"Logstash Airways",
"Montreal / Pierre Elliott Trudeau International Airport",
"YUL",
"Montreal",
"CA",
{
"lat": "45.47060013",
"lon": "-73.74079895"
},
"CA-QC",
"Heavy Fog",
10598.641992305174,
6585.690810855338,
0,
0,
"No Delay",
"JQGOQ0J",
9.2970543792150657,
557.82326275290393,
"Incheon International Airport",
"ICN",
"Seoul",
"KR",
{
"lat": "37.46910095",
"lon": "126.4509964"
},
"SE-BD",
"Hail",
6,
"2018-02-11T23:04:08Z"
],
[
788.63906303459316,
0,
"Kibana Airlines",
"London Gatwick Airport",
"LGW",
"London",
"GB",
{
"lat": "51.14810181",
"lon": "-0.190277994"
},
"GB-ENG",
"Sunny",
1407.2159452218614,
874.40344961789481,
0,
0,
"No Delay",
"VAI7N23",
1.6752570776450733,
100.51542465870439,
"Leonardo da Vinci - Fiumicino Airport",
"FCO",
"Rome",
"IT",
{
"lat": "41.8002778",
"lon": "12.2388889"
},
"SE-BD",
"Clear",
6,
"2018-02-11T23:01:03Z"
],
[
1155.3405895759547,
0,
"Kibana Airlines",
"OR Tambo International Airport",
"JNB",
"Johannesburg",
"ZA",
{
"lat": "-26.1392",
"lon": "28.246"
},
"SE-BD",
"Sunny",
11657.089629455511,
7243.3796810722324,
0,
0,
"No Delay",
"LO0I4OG",
14.944986704430143,
896.69920226580859,
"Brisbane International Airport",
"BNE",
"Brisbane",
"AU",
{
"lat": "-27.38419914",
"lon": "153.1170044"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T22:58:25Z"
],
[
929.83479006014068,
0,
"Logstash Airways",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Rain",
291.90172192067166,
181.37932096597845,
0,
0,
"No Delay",
"GR9Y0H6",
0.27027937214877007,
16.216762328926205,
"Verona Villafranca Airport",
"VR10",
"Verona",
"IT",
{
"lat": "45.395699",
"lon": "10.8885"
},
"IT-34",
"Cloudy",
6,
"2018-02-11T22:53:35Z"
],
[
842.29769420000446,
0,
"Kibana Airlines",
"Gimpo International Airport",
"GMP",
"Seoul",
"KR",
{
"lat": "37.5583",
"lon": "126.791"
},
"SE-BD",
"Clear",
7696.8683735093864,
4782.6122777413566,
0,
0,
"No Delay",
"SF9YY0W",
11.661921778044524,
699.71530668267144,
"Brisbane International Airport",
"BNE",
"Brisbane",
"AU",
{
"lat": "-27.38419914",
"lon": "153.1170044"
},
"SE-BD",
"Heavy Fog",
6,
"2018-02-11T22:44:14Z"
],
[
707.85021485573384,
0,
"Kibana Airlines",
"Cagliari Elmas Airport",
"CA07",
"Cagliari",
"IT",
{
"lat": "39.251499",
"lon": "9.05428"
},
"IT-88",
"Sunny",
7582.8660844200194,
4711.7745394521116,
0,
0,
"No Delay",
"IBSRCT5",
7.8988188379375206,
473.92913027625121,
"Austin Straubel International Airport",
"GRB",
"Green Bay",
"US",
{
"lat": "44.48509979",
"lon": "-88.12960052"
},
"US-WI",
"Clear",
6,
"2018-02-11T22:40:56Z"
],
[
1034.3357332067189,
0,
"ES-Air",
"Kempegowda International Airport",
"BLR",
"Bangalore",
"IN",
{
"lat": "13.1979",
"lon": "77.706299"
},
"SE-BD",
"Damaging Wind",
6248.0121637224674,
3882.3347672855939,
0,
0,
"No Delay",
"UXUB9IV",
8.67779467183676,
520.66768031020558,
"Itami Airport",
"ITM",
"Osaka",
"JP",
{
"lat": "34.78549957",
"lon": "135.4380035"
},
"SE-BD",
"Cloudy",
6,
"2018-02-11T22:17:15Z"
],
[
1077.0333368503507,
0,
"Kibana Airlines",
"Pisa International Airport",
"PI05",
"Pisa",
"IT",
{
"lat": "43.683899",
"lon": "10.3927"
},
"IT-52",
"Sunny",
7058.14624061129,
4385.7287445140937,
0,
0,
"No Delay",
"4WKID46",
9.04890543668114,
542.93432620086844,
"Norfolk International Airport",
"ORF",
"Norfolk",
"US",
{
"lat": "36.89459991",
"lon": "-76.20120239"
},
"US-VA",
"Thunder & Lightning",
6,
"2018-02-11T22:15:05Z"
],
[
294.55761631493806,
0,
"ES-Air",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Clear",
0.0,
0.0,
1,
105,
"Security Delay",
"MIK9FQC",
1.75,
105.0,
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Clear",
6,
"2018-02-11T22:06:14Z"
],
[
839.02205064681084,
0,
"ES-Air",
"Stockholm-Arlanda Airport",
"ARN",
"Stockholm",
"SE",
{
"lat": "59.65190125",
"lon": "17.91860008"
},
"SE-AB",
"Rain",
1223.803785060335,
760.4364169875023,
0,
0,
"No Delay",
"MPU4VJH",
1.1998076324120932,
71.9884579447256,
"Sheremetyevo International Airport",
"SVO",
"Moscow",
"RU",
{
"lat": "55.972599",
"lon": "37.4146"
},
"RU-MOS",
"Clear",
6,
"2018-02-11T22:06:14Z"
],
[
1067.5172474339875,
0,
"Logstash Airways",
"Leonardo da Vinci___Fiumicino Airport",
"RM11",
"Rome",
"IT",
{
"lat": "41.8002778",
"lon": "12.2388889"
},
"IT-62",
"Clear",
9813.843354728655,
6098.0395457581808,
1,
270,
"Late Aircraft Delay",
"D940ONX",
14.722753494509016,
883.36520967054093,
"Chubu Centrair International Airport",
"NGO",
"Tokoname",
"JP",
{
"lat": "34.85839844",
"lon": "136.8049927"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T21:44:41Z"
],
[
953.12119557978758,
0,
"Logstash Airways",
"New Chitose Airport",
"CTS",
"Chitose / Tomakomai",
"JP",
{
"lat": "42.77519989",
"lon": "141.6920013"
},
"SE-BD",
"Clear",
17010.351663163041,
10569.742493315935,
0,
0,
"No Delay",
"7WHK95X",
21.808143157901334,
1308.48858947408,
"Comodoro Arturo Merino Benitez International Airport",
"SCL",
"Santiago",
"CL",
{
"lat": "-33.39300156",
"lon": "-70.78579712"
},
"SE-BD",
"Clear",
6,
"2018-02-11T21:41:38Z"
],
[
1188.0474367030561,
0,
"Kibana Airlines",
"Sydney Kingsford Smith International Airport",
"SYD",
"Sydney",
"AU",
{
"lat": "-33.94609833",
"lon": "151.177002"
},
"SE-BD",
"Rain",
16609.582803034431,
10320.716268886223,
0,
0,
"No Delay",
"DHGBL3D",
21.294336926967219,
1277.6602156180331,
"Cagliari Elmas Airport",
"CA07",
"Cagliari",
"IT",
{
"lat": "39.251499",
"lon": "9.05428"
},
"IT-88",
"Damaging Wind",
6,
"2018-02-11T21:39:01Z"
],
[
444.73316181585278,
0,
"ES-Air",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Clear",
0.0,
0.0,
0,
0,
"No Delay",
"2DF7FDQ",
0.0,
0.0,
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Sunny",
6,
"2018-02-11T21:16:59Z"
],
[
514.763998895086,
0,
"ES-Air",
"Naples International Airport",
"NA01",
"Naples",
"IT",
{
"lat": "40.886002",
"lon": "14.2908"
},
"IT-72",
"Hail",
4155.0468641243951,
2581.826423762971,
0,
0,
"No Delay",
"RT11C1Y",
3.4625390534369962,
207.75234320621976,
"Abu Dhabi International Airport",
"AUH",
"Abu Dhabi",
"AE",
{
"lat": "24.43300056",
"lon": "54.65110016"
},
"SE-BD",
"Rain",
6,
"2018-02-11T20:58:57Z"
],
[
355.32975259018224,
0,
"Kibana Airlines",
"Ciampino___G. B. Pastine International Airport",
"RM12",
"Rome",
"IT",
{
"lat": "41.7994",
"lon": "12.5949"
},
"IT-62",
"Sunny",
0.0,
0.0,
0,
0,
"No Delay",
"Z2Q0URG",
0.0,
0.0,
"Ciampino___G. B. Pastine International Airport",
"RM12",
"Rome",
"IT",
{
"lat": "41.7994",
"lon": "12.5949"
},
"IT-62",
"Rain",
6,
"2018-02-11T20:55:06Z"
],
[
1181.5361621116438,
1,
"ES-Air",
"Mariscal Sucre International Airport",
"UIO",
"Quito",
"EC",
{
"lat": "-0.129166667",
"lon": "-78.3575"
},
"EC-P",
"Heavy Fog",
9799.8133772472956,
6089.3217219235257,
0,
0,
"No Delay",
"X19GQ3B",
8.16651114770608,
489.99066886236477,
"Frankfurt am Main Airport",
"FRA",
"Frankfurt am Main",
"DE",
{
"lat": "50.033333",
"lon": "8.570556"
},
"DE-HE",
"Sunny",
6,
"2018-02-11T20:55:06Z"
],
[
375.54305249369128,
0,
"ES-Air",
"Vienna International Airport",
"VIE",
"Vienna",
"AT",
{
"lat": "48.11029816",
"lon": "16.56970024"
},
"AT-9",
"Damaging Wind",
0.0,
0.0,
1,
360,
"Weather Delay",
"MX0N754",
6.0,
360.0,
"Vienna International Airport",
"VIE",
"Vienna",
"AT",
{
"lat": "48.11029816",
"lon": "16.56970024"
},
"AT-9",
"Heavy Fog",
6,
"2018-02-11T20:53:44Z"
],
[
867.231028821183,
0,
"ES-Air",
"Shanghai Pudong International Airport",
"PVG",
"Shanghai",
"CN",
{
"lat": "31.14340019",
"lon": "121.8050003"
},
"SE-BD",
"Damaging Wind",
9287.1298633138267,
5770.75495563026,
0,
0,
"No Delay",
"WK1D6D7",
8.5991943178831729,
515.95165907299042,
"Charles de Gaulle International Airport",
"CDG",
"Paris",
"FR",
{
"lat": "49.01279831",
"lon": "2.549999952"
},
"FR-J",
"Clear",
6,
"2018-02-11T20:51:31Z"
],
[
1080.4462785162759,
0,
"Logstash Airways",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Rain",
8058.5817527533545,
5007.3705514503763,
0,
0,
"No Delay",
"I0G6LLT",
6.7154847939611289,
402.92908763766775,
"Pisa International Airport",
"PI05",
"Pisa",
"IT",
{
"lat": "43.683899",
"lon": "10.3927"
},
"IT-52",
"Sunny",
6,
"2018-02-11T20:42:25Z"
],
[
511.71995921233889,
0,
"ES-Air",
"Cologne Bonn Airport",
"CGN",
"Cologne",
"DE",
{
"lat": "50.86589813",
"lon": "7.142739773"
},
"DE-NW",
"Clear",
769.26031174690661,
477.99619705103851,
1,
195,
"NAS Delay",
"TZHHKZ1",
3.8910502597890888,
233.46301558734533,
"Bologna Guglielmo Marconi Airport",
"BO08",
"Bologna",
"IT",
{
"lat": "44.5354",
"lon": "11.2887"
},
"IT-45",
"Cloudy",
6,
"2018-02-11T20:30:08Z"
],
[
602.62088547217445,
0,
"Kibana Airlines",
"San Diego International Airport",
"SAN",
"San Diego",
"US",
{
"lat": "32.73360062",
"lon": "-117.1900024"
},
"US-CA",
"Rain",
13618.993094316242,
8462.4499760873,
0,
0,
"No Delay",
"3A1PBBE",
11.946485170452842,
716.78911022717057,
"Abu Dhabi International Airport",
"AUH",
"Abu Dhabi",
"AE",
{
"lat": "24.43300056",
"lon": "54.65110016"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T20:24:22Z"
],
[
577.72386784829507,
0,
"Logstash Airways",
"San Antonio International Airport",
"SAT",
"San Antonio",
"US",
{
"lat": "29.53370094",
"lon": "-98.46980286"
},
"US-TX",
"Rain",
10505.11710358143,
6527.5771392452007,
0,
0,
"No Delay",
"F6OPZ0P",
9.2150150031416054,
552.90090018849628,
"Ataturk International Airport",
"IST",
"Istanbul",
"TR",
{
"lat": "40.97689819",
"lon": "28.81459999"
},
"TR-34",
"Sunny",
6,
"2018-02-11T20:19:55Z"
],
[
795.90527761085707,
0,
"Kibana Airlines",
"Malpensa International Airport",
"MI12",
"Milan",
"IT",
{
"lat": "45.6306",
"lon": "8.72811"
},
"IT-25",
"Sunny",
9618.7648755421524,
5976.8233985662182,
0,
0,
"No Delay",
"9D9X8N4",
8.9062637736501422,
534.37582641900849,
"Itami Airport",
"ITM",
"Osaka",
"JP",
{
"lat": "34.78549957",
"lon": "135.4380035"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T20:10:13Z"
],
[
1078.6406790975925,
0,
"Logstash Airways",
"Jorge Chavez International Airport",
"LIM",
"Lima",
"PE",
{
"lat": "-12.0219",
"lon": "-77.114304"
},
"SE-BD",
"Sunny",
10511.438749435607,
6531.5052278665135,
1,
225,
"Late Aircraft Delay",
"652J76O",
17.226203524917445,
1033.5722114950468,
"Amsterdam Airport Schiphol",
"AMS",
"Amsterdam",
"NL",
{
"lat": "52.30860138",
"lon": "4.76388979"
},
"NL-NH",
"Rain",
6,
"2018-02-11T20:08:25Z"
],
[
840.46119039508471,
0,
"Kibana Airlines",
"Ukrainka Air Base",
"XHBU",
"Belogorsk",
"RU",
{
"lat": "51.169997",
"lon": "128.445007"
},
"RU-AMU",
"Rain",
2294.2233422135751,
1425.56429340997,
1,
45,
"NAS Delay",
"O9N0WNY",
3.139815981472474,
188.38895888834844,
"Shanghai Hongqiao International Airport",
"SHA",
"Shanghai",
"CN",
{
"lat": "31.19790077",
"lon": "121.3359985"
},
"SE-BD",
"Rain",
6,
"2018-02-11T20:03:31Z"
],
[
891.14456428013091,
0,
"Logstash Airways",
"Jeju International Airport",
"CJU",
"Jeju City",
"KR",
{
"lat": "33.51129913",
"lon": "126.4929962"
},
"SE-BD",
"Rain",
836.66289131553981,
519.87821827747189,
0,
0,
"No Delay",
"5MK5DV7",
0.6972190760962832,
41.83314456577699,
"Itami Airport",
"ITM",
"Osaka",
"JP",
{
"lat": "34.78549957",
"lon": "135.4380035"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T20:03:31Z"
],
[
313.42919532459871,
0,
"JetBeats",
"Tucson International Airport",
"TUS",
"Tucson",
"US",
{
"lat": "32.11610031",
"lon": "-110.9410019"
},
"US-AZ",
"Cloudy",
3057.7707791247972,
1900.0106746132567,
0,
0,
"No Delay",
"HNEWZ0B",
3.1851778949216638,
191.11067369529982,
"Miami International Airport",
"MIA",
"Miami",
"US",
{
"lat": "25.79319954",
"lon": "-80.29060364"
},
"US-FL",
"Sunny",
6,
"2018-02-11T19:54:11Z"
],
[
1053.5419730483895,
0,
"JetBeats",
"Shanghai Hongqiao International Airport",
"SHA",
"Shanghai",
"CN",
{
"lat": "31.19790077",
"lon": "121.3359985"
},
"SE-BD",
"Rain",
15994.086504249335,
9938.2645998924618,
0,
0,
"No Delay",
"H3UL18F",
17.771207226943705,
1066.2724336166223,
"Mariscal Sucre International Airport",
"UIO",
"Quito",
"EC",
{
"lat": "-0.129166667",
"lon": "-78.3575"
},
"SE-BD",
"Cloudy",
6,
"2018-02-11T19:54:11Z"
],
[
684.9300961506799,
0,
"JetBeats",
"Vienna International Airport",
"VIE",
"Vienna",
"AT",
{
"lat": "48.11029816",
"lon": "16.56970024"
},
"AT-9",
"Cloudy",
768.002930319632,
477.21489645447582,
0,
0,
"No Delay",
"TPA2HJK",
0.64000244193302669,
38.4001465159816,
"Ciampino___G. B. Pastine International Airport",
"RM12",
"Rome",
"IT",
{
"lat": "41.7994",
"lon": "12.5949"
},
"IT-62",
"Heavy Fog",
6,
"2018-02-11T19:51:23Z"
],
[
771.28019232022439,
0,
"Kibana Airlines",
"Cagliari Elmas Airport",
"CA07",
"Cagliari",
"IT",
{
"lat": "39.251499",
"lon": "9.05428"
},
"IT-88",
"Hail",
16456.262353521666,
10225.447358378113,
0,
0,
"No Delay",
"DYQ1H18",
13.060525677398148,
783.63154064388891,
"Brisbane International Airport",
"BNE",
"Brisbane",
"AU",
{
"lat": "-27.38419914",
"lon": "153.1170044"
},
"SE-BD",
"Heavy Fog",
6,
"2018-02-11T19:40:16Z"
],
[
680.78652721915194,
0,
"JetBeats",
"Edmonton International Airport",
"CYEG",
"Edmonton",
"CA",
{
"lat": "53.30970001",
"lon": "-113.5800018"
},
"CA-AB",
"Cloudy",
8953.6337464276367,
5563.5300758741678,
0,
0,
"No Delay",
"MRGS1S6",
9.3267018191954545,
559.60210915172729,
"Ataturk International Airport",
"IST",
"Istanbul",
"TR",
{
"lat": "40.97689819",
"lon": "28.81459999"
},
"TR-34",
"Rain",
6,
"2018-02-11T19:34:39Z"
],
[
1191.0429351122139,
1,
"ES-Air",
"Munich Airport",
"MUC",
"Munich",
"DE",
{
"lat": "48.353802",
"lon": "11.7861"
},
"DE-BY",
"Heavy Fog",
8902.9489415275839,
5532.0359982251048,
1,
285,
"NAS Delay",
"7ZS3Z50",
17.115206863232757,
1026.9124117939655,
"Guangzhou Baiyun International Airport",
"CAN",
"Guangzhou",
"CN",
{
"lat": "23.39240074",
"lon": "113.2990036"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T19:21:16Z"
],
[
294.710648297039,
0,
"JetBeats",
"Treviso-Sant'Angelo Airport",
"TV01",
"Treviso",
"IT",
{
"lat": "45.648399",
"lon": "12.1944"
},
"IT-34",
"Clear",
0.0,
0.0,
1,
60,
"Carrier Delay",
"HKKLFK0",
1.0,
60.0,
"Treviso-Sant'Angelo Airport",
"TV01",
"Treviso",
"IT",
{
"lat": "45.648399",
"lon": "12.1944"
},
"IT-34",
"Clear",
6,
"2018-02-11T19:17:53Z"
],
[
1191.9641043265337,
0,
"Logstash Airways",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Hail",
5899.45446531149,
3665.751054660464,
1,
360,
"Late Aircraft Delay",
"ZMOKFBY",
12.145265068032803,
728.71590408196812,
"Portland International Jetport Airport",
"PWM",
"Portland",
"US",
{
"lat": "43.64619827",
"lon": "-70.30930328"
},
"US-ME",
"Clear",
6,
"2018-02-11T19:02:10Z"
],
[
863.38806817372051,
0,
"Logstash Airways",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Damaging Wind",
2823.4526611229717,
1754.4121462676542,
0,
0,
"No Delay",
"HSO3VP2",
2.3528772176024764,
141.17263305614858,
"Tokyo Haneda International Airport",
"HND",
"Tokyo",
"JP",
{
"lat": "35.552299",
"lon": "139.779999"
},
"SE-BD",
"Clear",
6,
"2018-02-11T18:59:53Z"
],
[
1049.4900036328584,
0,
"Logstash Airways",
"Portland International Airport",
"PDX",
"Portland",
"US",
{
"lat": "45.58869934",
"lon": "-122.5979996"
},
"US-OR",
"Damaging Wind",
3617.749726683272,
2247.9654608854735,
1,
300,
"Late Aircraft Delay",
"LI2IKZC",
8.1734646725291853,
490.40788035175115,
"Licenciado Benito Juarez International Airport",
"AICM",
"Mexico City",
"MX",
{
"lat": "19.4363",
"lon": "-99.072098"
},
"MX-DIF",
"Cloudy",
6,
"2018-02-11T18:40:21Z"
],
[
1085.1553392784897,
0,
"Logstash Airways",
"Melbourne International Airport",
"MEL",
"Melbourne",
"AU",
{
"lat": "-37.673302",
"lon": "144.843002"
},
"SE-BD",
"Cloudy",
16100.120197134849,
10004.15088205806,
1,
150,
"Weather Delay",
"O4A214N",
17.407518701050783,
1044.451122063047,
"Bologna Guglielmo Marconi Airport",
"BO08",
"Bologna",
"IT",
{
"lat": "44.5354",
"lon": "11.2887"
},
"IT-45",
"Cloudy",
6,
"2018-02-11T18:35:42Z"
],
[
920.60641577916419,
1,
"Logstash Airways",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Sunny",
7446.4285318079565,
4626.99617471961,
0,
0,
"No Delay",
"OXT7J51",
8.8647958711999486,
531.88775227199687,
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Hail",
6,
"2018-02-11T18:28:07Z"
],
[
728.72199250374842,
0,
"ES-Air",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Sunny",
12994.258295607915,
8074.2577693817566,
0,
0,
"No Delay",
"WLGAAS9",
13.535685724591579,
812.14114347549469,
"Tampa International Airport",
"TPA",
"Tampa",
"US",
{
"lat": "27.97550011",
"lon": "-82.53320313"
},
"US-FL",
"Clear",
6,
"2018-02-11T18:28:07Z"
],
[
748.91132012619278,
0,
"Logstash Airways",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Clear",
10316.155866415009,
6410.16207002046,
0,
0,
"No Delay",
"RXAWAEN",
10.113878300406871,
606.83269802441225,
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Thunder & Lightning",
6,
"2018-02-11T18:24:16Z"
],
[
1190.114570302957,
0,
"ES-Air",
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Sunny",
2798.6231896448967,
1738.9838279726998,
0,
0,
"No Delay",
"368L67X",
3.587978448262688,
215.27870689576128,
"Olenya Air Base",
"XLMO",
"Olenegorsk",
"RU",
{
"lat": "68.15180206",
"lon": "33.46390152"
},
"RU-MUR",
"Sunny",
6,
"2018-02-11T18:19:33Z"
],
[
897.39705131952633,
0,
"Logstash Airways",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Sunny",
261.002285347155,
162.17930122283053,
0,
0,
"No Delay",
"R65C8M6",
0.20714467091044048,
12.428680254626428,
"Turin Airport",
"TO11",
"Torino",
"IT",
{
"lat": "45.200802",
"lon": "7.64963"
},
"IT-21",
"Damaging Wind",
6,
"2018-02-11T18:18:07Z"
],
[
1073.8100029687585,
1,
"ES-Air",
"Verona Villafranca Airport",
"VR10",
"Verona",
"IT",
{
"lat": "45.395699",
"lon": "10.8885"
},
"IT-34",
"Damaging Wind",
9093.6165223126191,
5650.5113402185107,
1,
195,
"Carrier Delay",
"HHWEFAG",
17.028206851988816,
1021.692411119329,
"New Chitose Airport",
"CTS",
"Chitose / Tomakomai",
"JP",
{
"lat": "42.77519989",
"lon": "141.6920013"
},
"SE-BD",
"Clear",
6,
"2018-02-11T18:13:09Z"
],
[
549.86184823154326,
0,
"JetBeats",
"Melbourne International Airport",
"MEL",
"Melbourne",
"AU",
{
"lat": "-37.673302",
"lon": "144.843002"
},
"SE-BD",
"Sunny",
9795.9122912271068,
6086.8976994521408,
0,
0,
"No Delay",
"FEBSZFG",
13.605433737815426,
816.32602426892561,
"Chhatrapati Shivaji International Airport",
"BOM",
"Mumbai",
"IN",
{
"lat": "19.08869934",
"lon": "72.86789703"
},
"SE-BD",
"Cloudy",
6,
"2018-02-11T18:12:20Z"
],
[
231.85512623777851,
0,
"ES-Air",
"New Chitose Airport",
"CTS",
"Chitose / Tomakomai",
"JP",
{
"lat": "42.77519989",
"lon": "141.6920013"
},
"SE-BD",
"Damaging Wind",
1039.4592136247936,
645.89001085211964,
0,
0,
"No Delay",
"MW2DSNT",
1.2374514447914211,
74.247086687485265,
"Itami Airport",
"ITM",
"Osaka",
"JP",
{
"lat": "34.78549957",
"lon": "135.4380035"
},
"SE-BD",
"Rain",
6,
"2018-02-11T18:06:11Z"
],
[
868.0747522456586,
1,
"ES-Air",
"Ukrainka Air Base",
"XHBU",
"Belogorsk",
"RU",
{
"lat": "51.169997",
"lon": "128.445007"
},
"RU-AMU",
"Heavy Fog",
13450.984211948124,
8358.0540965437613,
1,
180,
"Security Delay",
"RBZEOFK",
14.799108957849231,
887.9465374709539,
"El Dorado International Airport",
"BOG",
"Bogota",
"CO",
{
"lat": "4.70159",
"lon": "-74.1469"
},
"SE-BD",
"Clear",
6,
"2018-02-11T17:37:26Z"
],
[
1019.3545756274339,
0,
"ES-Air",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Thunder & Lightning",
7290.3553136709525,
4530.0167730895018,
1,
45,
"NAS Delay",
"CEBRVZV",
8.8503947929677249,
531.02368757806346,
"Milano Linate Airport",
"MI11",
"Milan",
"IT",
{
"lat": "45.445099",
"lon": "9.27674"
},
"IT-25",
"Clear",
6,
"2018-02-11T17:23:45Z"
],
[
730.44248977898883,
1,
"Logstash Airways",
"Ministro Pistarini International Airport",
"EZE",
"Buenos Aires",
"AR",
{
"lat": "-34.8222",
"lon": "-58.5358"
},
"AR-B",
"Sunny",
5947.2054673771754,
3695.4221517445462,
0,
0,
"No Delay",
"1B1K5HI",
7.0800065087823514,
424.8003905269411,
"Luis Munoz Marin International Airport",
"SJU",
"San Juan",
"PR",
{
"lat": "18.43939972",
"lon": "-66.00180054"
},
"PR-U-A",
"Damaging Wind",
6,
"2018-02-11T17:09:05Z"
],
[
684.55612753345918,
0,
"Logstash Airways",
"Sydney Kingsford Smith International Airport",
"SYD",
"Sydney",
"AU",
{
"lat": "-33.94609833",
"lon": "151.177002"
},
"SE-BD",
"Hail",
15587.644267855676,
9685.7131028889253,
1,
360,
"Security Delay",
"K7XY3E3",
29.61764283008436,
1777.0585698050616,
"Stockholm-Arlanda Airport",
"ARN",
"Stockholm",
"SE",
{
"lat": "59.65190125",
"lon": "17.91860008"
},
"SE-AB",
"Sunny",
6,
"2018-02-11T17:07:09Z"
],
[
841.89072025689461,
1,
"JetBeats",
"Treviso-Sant'Angelo Airport",
"TV01",
"Treviso",
"IT",
{
"lat": "45.648399",
"lon": "12.1944"
},
"IT-34",
"Cloudy",
8185.8556810231057,
5086.45490400008,
0,
0,
"No Delay",
"XPX46W4",
11.369244001420981,
682.15464008525885,
"Tampa International Airport",
"TPA",
"Tampa",
"US",
{
"lat": "27.97550011",
"lon": "-82.53320313"
},
"US-FL",
"Hail",
6,
"2018-02-11T17:04:12Z"
],
[
355.61575230647952,
0,
"Kibana Airlines",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Sunny",
0.0,
0.0,
1,
165,
"Carrier Delay",
"C32CL1J",
2.75,
165.0,
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Hail",
6,
"2018-02-11T16:57:01Z"
],
[
1012.6620251058478,
0,
"Kibana Airlines",
"El Dorado International Airport",
"BOG",
"Bogota",
"CO",
{
"lat": "4.70159",
"lon": "-74.1469"
},
"CO-CUN",
"Rain",
9746.3269863838959,
6056.0868194642635,
0,
0,
"No Delay",
"FMJP7A2",
7.7351801479237272,
464.11080887542363,
"Bari Karol Wojty__a Airport",
"BA02",
"Bari",
"IT",
{
"lat": "41.138901",
"lon": "16.760599"
},
"IT-75",
"Rain",
6,
"2018-02-11T16:40:54Z"
],
[
925.96759191344518,
0,
"JetBeats",
"Narita International Airport",
"NRT",
"Tokyo",
"JP",
{
"lat": "35.76470184",
"lon": "140.3860016"
},
"SE-BD",
"Hail",
9636.2454993724077,
5987.6853546366765,
0,
0,
"No Delay",
"AG0QLCR",
8.4528469292740418,
507.17081575644249,
"London Gatwick Airport",
"LGW",
"London",
"GB",
{
"lat": "51.14810181",
"lon": "-0.190277994"
},
"GB-ENG",
"Clear",
6,
"2018-02-11T16:23:20Z"
],
[
524.39622986737254,
1,
"ES-Air",
"Turin Airport",
"TO11",
"Torino",
"IT",
{
"lat": "45.200802",
"lon": "7.64963"
},
"IT-21",
"Damaging Wind",
6220.0271998065373,
3864.9457168924337,
0,
0,
"No Delay",
"4FAPIOY",
8.6389266663979676,
518.33559998387807,
"Bradley International Airport",
"BDL",
"Hartford",
"US",
{
"lat": "41.93889999",
"lon": "-72.68319702"
},
"US-CT",
"Clear",
6,
"2018-02-11T16:16:10Z"
],
[
1122.9557137584175,
0,
"Logstash Airways",
"Montreal / Pierre Elliott Trudeau International Airport",
"YUL",
"Montreal",
"CA",
{
"lat": "45.47060013",
"lon": "-73.74079895"
},
"CA-QC",
"Sunny",
7733.56632835579,
4805.415329696938,
1,
30,
"Late Aircraft Delay",
"Y18SAKV",
10.414828626097167,
624.88971756583,
"Ataturk International Airport",
"IST",
"Istanbul",
"TR",
{
"lat": "40.97689819",
"lon": "28.81459999"
},
"TR-34",
"Rain",
6,
"2018-02-11T16:16:10Z"
],
[
538.72090343251773,
0,
"ES-Air",
"London Luton Airport",
"LTN",
"London",
"GB",
{
"lat": "51.87469864",
"lon": "-0.368333012"
},
"GB-ENG",
"Sunny",
16505.067926876196,
10255.773735681243,
0,
0,
"No Delay",
"X2V7XWN",
17.192779090496039,
1031.5667454297623,
"Brisbane International Airport",
"BNE",
"Brisbane",
"AU",
{
"lat": "-27.38419914",
"lon": "153.1170044"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T16:09:59Z"
],
[
921.64888827543177,
0,
"ES-Air",
"Leonardo da Vinci - Fiumicino Airport",
"FCO",
"Rome",
"IT",
{
"lat": "41.8002778",
"lon": "12.2388889"
},
"SE-BD",
"Rain",
9742.7708213442475,
6053.8771209537845,
0,
0,
"No Delay",
"NSJQKWC",
14.761773971733708,
885.70643830402253,
"Itami Airport",
"ITM",
"Osaka",
"JP",
{
"lat": "34.78549957",
"lon": "135.4380035"
},
"SE-BD",
"Damaging Wind",
6,
"2018-02-11T16:07:20Z"
],
[
1136.6781500052271,
1,
"Kibana Airlines",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Heavy Fog",
2264.78763216076,
1407.2737911601,
1,
345,
"Late Aircraft Delay",
"62IWL46",
7.736655817684877,
464.19934906109262,
"San Antonio International Airport",
"SAT",
"San Antonio",
"US",
{
"lat": "29.53370094",
"lon": "-98.46980286"
},
"US-TX",
"Cloudy",
6,
"2018-02-11T16:03:10Z"
],
[
605.19187610775509,
0,
"JetBeats",
"Portland International Jetport Airport",
"PWM",
"Portland",
"US",
{
"lat": "43.64619827",
"lon": "-70.30930328"
},
"US-ME",
"Thunder & Lightning",
11291.997145070151,
7016.52172877281,
0,
0,
"No Delay",
"SEFB5XW",
9.4099976208917919,
564.59985725350748,
"Jeju International Airport",
"CJU",
"Jeju City",
"KR",
{
"lat": "33.51129913",
"lon": "126.4929962"
},
"SE-BD",
"Cloudy",
6,
"2018-02-11T15:55:10Z"
],
[
892.0331086545516,
0,
"ES-Air",
"London Heathrow Airport",
"LHR",
"London",
"GB",
{
"lat": "51.4706",
"lon": "-0.461941"
},
"GB-ENG",
"Rain",
347.65282630016048,
216.02145116280948,
0,
0,
"No Delay",
"0ZEBVTZ",
0.28971068858346705,
17.382641315008023,
"Charles de Gaulle International Airport",
"CDG",
"Paris",
"FR",
{
"lat": "49.01279831",
"lon": "2.549999952"
},
"FR-J",
"Cloudy",
6,
"2018-02-11T15:41:06Z"
],
[
529.71654815317549,
0,
"ES-Air",
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Rain",
7056.7930552279513,
4384.88791409913,
0,
0,
"No Delay",
"6G8EGS6",
6.5340676437295846,
392.04405862377507,
"Kempegowda International Airport",
"BLR",
"Bangalore",
"IN",
{
"lat": "13.1979",
"lon": "77.706299"
},
"SE-BD",
"Cloudy",
6,
"2018-02-11T15:41:06Z"
],
[
1136.0959442550548,
0,
"ES-Air",
"Tokyo Haneda International Airport",
"HND",
"Tokyo",
"JP",
{
"lat": "35.552299",
"lon": "139.779999"
},
"SE-BD",
"Damaging Wind",
10899.138483601675,
6772.41067391538,
0,
0,
"No Delay",
"MG6614J",
9.5606477926330466,
573.63886755798285,
"John F Kennedy International Airport",
"JFK",
"New York",
"US",
{
"lat": "40.63980103",
"lon": "-73.77890015"
},
"US-NY",
"Thunder & Lightning",
6,
"2018-02-11T15:27:27Z"
],
[
480.45632044880244,
0,
"ES-Air",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Clear",
0.0,
0.0,
1,
270,
"Late Aircraft Delay",
"I9WOJ68",
4.5,
270.0,
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Clear",
6,
"2018-02-11T15:27:27Z"
],
[
429.58053887313531,
0,
"Logstash Airways",
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Sunny",
0.0,
0.0,
1,
150,
"Late Aircraft Delay",
"I2IGGNL",
2.5,
150.0,
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Cloudy",
6,
"2018-02-11T15:07:11Z"
],
[
858.14433690388387,
0,
"JetBeats",
"Washington Dulles International Airport",
"IAD",
"Washington",
"US",
{
"lat": "38.94449997",
"lon": "-77.45580292"
},
"US-DC",
"Heavy Fog",
16809.1419228649,
10444.716557097114,
1,
210,
"NAS Delay",
"BJ4UMEM",
26.84603044842347,
1610.7618269054083,
"Adelaide International Airport",
"ADL",
"Adelaide",
"AU",
{
"lat": "-34.945",
"lon": "138.531006"
},
"SE-BD",
"Rain",
6,
"2018-02-11T14:54:34Z"
],
[
1184.1554070522866,
1,
"JetBeats",
"Warsaw Chopin Airport",
"WAW",
"Warsaw",
"PL",
{
"lat": "52.16569901",
"lon": "20.96710014"
},
"PL-MZ",
"Hail",
1246.3042852492376,
774.41757961581709,
0,
0,
"No Delay",
"JQN64EB",
1.2982336304679558,
77.894017828077352,
"Turin Airport",
"TO11",
"Torino",
"IT",
{
"lat": "45.200802",
"lon": "7.64963"
},
"IT-21",
"Clear",
6,
"2018-02-11T14:40:05Z"
],
[
587.04819736377317,
0,
"JetBeats",
"Ottawa Macdonald-Cartier International Airport",
"YOW",
"Ottawa",
"CA",
{
"lat": "45.32249832",
"lon": "-75.66919708"
},
"CA-ON",
"Sunny",
6266.5996814090468,
3893.8845153112366,
0,
0,
"No Delay",
"VAHCDKK",
6.1437251778520068,
368.62351067112041,
"Malpensa International Airport",
"MI12",
"Milan",
"IT",
{
"lat": "45.6306",
"lon": "8.72811"
},
"IT-25",
"Sunny",
6,
"2018-02-11T14:39:28Z"
],
[
861.32564969494854,
0,
"Logstash Airways",
"King Shaka International Airport",
"DUR",
"Durban",
"ZA",
{
"lat": "-29.61444444",
"lon": "31.11972222"
},
"SE-BD",
"Cloudy",
9221.41433967435,
5729.9212223578979,
0,
0,
"No Delay",
"RE0BZ6S",
8.0889599470827633,
485.33759682496577,
"Cologne Bonn Airport",
"CGN",
"Cologne",
"DE",
{
"lat": "50.86589813",
"lon": "7.142739773"
},
"DE-NW",
"Cloudy",
6,
"2018-02-11T14:32:13Z"
],
[
1007.3040547218907,
0,
"Kibana Airlines",
"Montreal / Pierre Elliott Trudeau International Airport",
"YUL",
"Montreal",
"CA",
{
"lat": "45.47060013",
"lon": "-73.74079895"
},
"CA-QC",
"Rain",
4517.5310449555327,
2807.0636513731883,
0,
0,
"No Delay",
"3CEKFS9",
5.791706467891709,
347.50238807350252,
"El Dorado International Airport",
"BOG",
"Bogota",
"CO",
{
"lat": "4.70159",
"lon": "-74.1469"
},
"SE-BD",
"Clear",
6,
"2018-02-11T14:31:39Z"
],
[
928.53109024437617,
1,
"ES-Air",
"Dublin Airport",
"DUB",
"Dublin",
"IE",
{
"lat": "53.42129898",
"lon": "-6.270070076"
},
"IE-D",
"Rain",
5942.7694626619987,
3692.6657462059065,
0,
0,
"No Delay",
"0M0SIGP",
4.9523078855516651,
297.13847313309992,
"Al Maktoum International Airport",
"DWC",
"Jebel Ali",
"AE",
{
"lat": "24.896356",
"lon": "55.161389"
},
"SE-BD",
"Thunder & Lightning",
6,
"2018-02-11T14:15:07Z"
],
[
655.38679477159587,
0,
"ES-Air",
"Rajiv Gandhi International Airport",
"HYD",
"Hyderabad",
"IN",
{
"lat": "17.23131752",
"lon": "78.42985535"
},
"SE-BD",
"Thunder & Lightning",
15343.038834032279,
9533.7223328463515,
0,
0,
"No Delay",
"M8D9B52",
15.042194935325764,
902.53169611954581,
"Ministro Pistarini International Airport",
"EZE",
"Buenos Aires",
"AR",
{
"lat": "-34.8222",
"lon": "-58.5358"
},
"AR-B",
"Cloudy",
6,
"2018-02-11T14:04:11Z"
],
[
1015.1401424754488,
0,
"Kibana Airlines",
"Sydney Kingsford Smith International Airport",
"SYD",
"Sydney",
"AU",
{
"lat": "-33.94609833",
"lon": "151.177002"
},
"SE-BD",
"Damaging Wind",
7900.4759567292895,
4909.12816447527,
0,
0,
"No Delay",
"KMTEHP9",
6.5837299639410753,
395.0237978364645,
"Jeju International Airport",
"CJU",
"Jeju City",
"KR",
{
"lat": "33.51129913",
"lon": "126.4929962"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T14:03:05Z"
],
[
759.34563886354022,
0,
"ES-Air",
"Turin Airport",
"TO11",
"Torino",
"IT",
{
"lat": "45.200802",
"lon": "7.64963"
},
"IT-21",
"Cloudy",
6221.0988725051557,
3865.6116234348624,
0,
0,
"No Delay",
"A2MIKJ8",
7.4060700863156619,
444.3642051789397,
"Ottawa Macdonald-Cartier International Airport",
"YOW",
"Ottawa",
"CA",
{
"lat": "45.32249832",
"lon": "-75.66919708"
},
"CA-ON",
"Heavy Fog",
6,
"2018-02-11T14:02:22Z"
],
[
999.02125642634246,
0,
"Kibana Airlines",
"Kempegowda International Airport",
"BLR",
"Bangalore",
"IN",
{
"lat": "13.1979",
"lon": "77.706299"
},
"SE-BD",
"Cloudy",
6721.4228198093351,
4176.498511076149,
1,
60,
"Security Delay",
"B2JWDRX",
8.0014821039680566,
480.08892623808345,
"Catania-Fontanarossa Airport",
"CT03",
"Catania",
"IT",
{
"lat": "37.466801",
"lon": "15.0664"
},
"IT-82",
"Hail",
6,
"2018-02-11T13:32:15Z"
],
[
990.38260659260914,
0,
"ES-Air",
"Narita International Airport",
"NRT",
"Tokyo",
"JP",
{
"lat": "35.76470184",
"lon": "140.3860016"
},
"SE-BD",
"Rain",
9614.5290987101871,
5974.1914088660888,
1,
300,
"Weather Delay",
"RX41Z27",
15.015134477823112,
900.90806866938669,
"Dublin Airport",
"DUB",
"Dublin",
"IE",
{
"lat": "53.42129898",
"lon": "-6.270070076"
},
"IE-D",
"Thunder & Lightning",
6,
"2018-02-11T13:31:33Z"
],
[
673.70318711783193,
0,
"ES-Air",
"Helsinki Vantaa Airport",
"HEL",
"Helsinki",
"FI",
{
"lat": "60.31719971",
"lon": "24.9633007"
},
"FI-ES",
"Sunny",
1900.0150603475022,
1180.6146233170175,
0,
0,
"No Delay",
"OKRHE5O",
2.4359167440352594,
146.15500464211556,
"Charles de Gaulle International Airport",
"CDG",
"Paris",
"FR",
{
"lat": "49.01279831",
"lon": "2.549999952"
},
"FR-J",
"Sunny",
6,
"2018-02-11T13:28:33Z"
],
[
628.88281473295581,
0,
"Logstash Airways",
"Montreal / Pierre Elliott Trudeau International Airport",
"YUL",
"Montreal",
"CA",
{
"lat": "45.47060013",
"lon": "-73.74079895"
},
"CA-QC",
"Rain",
12345.581188715831,
7671.1885020951577,
0,
0,
"No Delay",
"3BKOUKB",
18.705426043508837,
1122.3255626105301,
"Guangzhou Baiyun International Airport",
"CAN",
"Guangzhou",
"CN",
{
"lat": "23.39240074",
"lon": "113.2990036"
},
"SE-BD",
"Thunder & Lightning",
6,
"2018-02-11T13:25:53Z"
],
[
1055.3502132890076,
1,
"Kibana Airlines",
"Warsaw Chopin Airport",
"WAW",
"Warsaw",
"PL",
{
"lat": "52.16569901",
"lon": "20.96710014"
},
"PL-MZ",
"Sunny",
1246.3042852492376,
774.41757961581709,
0,
0,
"No Delay",
"OIOFJLD",
1.1539854493048496,
69.239126958290981,
"Turin Airport",
"TO11",
"Torino",
"IT",
{
"lat": "45.200802",
"lon": "7.64963"
},
"IT-21",
"Thunder & Lightning",
6,
"2018-02-11T13:20:16Z"
],
[
807.73261475866343,
0,
"ES-Air",
"Cagliari Elmas Airport",
"CA07",
"Cagliari",
"IT",
{
"lat": "39.251499",
"lon": "9.05428"
},
"IT-88",
"Sunny",
15564.831100919582,
9671.5376581511355,
1,
285,
"Late Aircraft Delay",
"M8R7LBZ",
28.333077425635732,
1699.9846455381439,
"Adelaide International Airport",
"ADL",
"Adelaide",
"AU",
{
"lat": "-34.945",
"lon": "138.531006"
},
"SE-BD",
"Cloudy",
6,
"2018-02-11T13:10:38Z"
],
[
777.47661678526742,
0,
"JetBeats",
"Tokyo Haneda International Airport",
"HND",
"Tokyo",
"JP",
{
"lat": "35.552299",
"lon": "139.779999"
},
"SE-BD",
"Clear",
11692.454203402607,
7265.3542085487043,
0,
0,
"No Delay",
"QZA8OCM",
14.990325901798213,
899.41955410789285,
"Orlando Sanford International Airport",
"SFB",
"Orlando",
"US",
{
"lat": "28.77759933",
"lon": "-81.23750305"
},
"US-FL",
"Thunder & Lightning",
6,
"2018-02-11T12:32:18Z"
],
[
594.0540183976791,
0,
"Kibana Airlines",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Rain",
3566.1857360188378,
2215.9250825297995,
1,
315,
"Carrier Delay",
"2NUO9LB",
9.8220329948959453,
589.3219796937567,
"Rajiv Gandhi International Airport",
"HYD",
"Hyderabad",
"IN",
{
"lat": "17.23131752",
"lon": "78.42985535"
},
"SE-BD",
"Clear",
6,
"2018-02-11T12:27:52Z"
],
[
861.38411356370273,
0,
"Logstash Airways",
"Stockholm-Arlanda Airport",
"ARN",
"Stockholm",
"SE",
{
"lat": "59.65190125",
"lon": "17.91860008"
},
"SE-AB",
"Sunny",
1466.2887108582138,
911.10956443011173,
1,
255,
"Weather Delay",
"2V5TSX6",
5.4719072590485114,
328.3144355429107,
"London Heathrow Airport",
"LHR",
"London",
"GB",
{
"lat": "51.4706",
"lon": "-0.461941"
},
"GB-ENG",
"Clear",
6,
"2018-02-11T12:24:58Z"
],
[
308.35517140080987,
0,
"Logstash Airways",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Sunny",
0.0,
0.0,
1,
300,
"Carrier Delay",
"PR96VG3",
5.0,
300.0,
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Clear",
6,
"2018-02-11T12:22:16Z"
],
[
994.35627144601767,
0,
"ES-Air",
"Narita International Airport",
"NRT",
"Tokyo",
"JP",
{
"lat": "35.76470184",
"lon": "140.3860016"
},
"SE-BD",
"Rain",
11271.561706708899,
7003.8237360743869,
0,
0,
"No Delay",
"HRXGJGK",
11.050550692851861,
663.03304157111165,
"Licenciado Benito Juarez International Airport",
"AICM",
"Mexico City",
"MX",
{
"lat": "19.4363",
"lon": "-99.072098"
},
"MX-DIF",
"Clear",
6,
"2018-02-11T12:13:55Z"
],
[
1175.9079216638825,
0,
"ES-Air",
"Ukrainka Air Base",
"XHBU",
"Belogorsk",
"RU",
{
"lat": "51.169997",
"lon": "128.445007"
},
"RU-AMU",
"Rain",
13450.984211948124,
8358.0540965437613,
0,
0,
"No Delay",
"WT4XS8N",
16.013076442795384,
960.78458656772307,
"El Dorado International Airport",
"BOG",
"Bogota",
"CO",
{
"lat": "4.70159",
"lon": "-74.1469"
},
"CO-CUN",
"Damaging Wind",
6,
"2018-02-11T12:07:32Z"
],
[
899.94727743659166,
0,
"Kibana Airlines",
"Narita International Airport",
"NRT",
"Tokyo",
"JP",
{
"lat": "35.76470184",
"lon": "140.3860016"
},
"SE-BD",
"Rain",
1296.9342910553842,
805.87760668656551,
1,
330,
"Carrier Delay",
"LC8WCRT",
6.70086508431054,
402.05190505863243,
"Jeju International Airport",
"CJU",
"Jeju City",
"KR",
{
"lat": "33.51129913",
"lon": "126.4929962"
},
"SE-BD",
"Rain",
6,
"2018-02-11T12:05:52Z"
],
[
817.36895217510494,
0,
"JetBeats",
"Syracuse Hancock International Airport",
"SYR",
"Syracuse",
"US",
{
"lat": "43.11119843",
"lon": "-76.10630035"
},
"US-NY",
"Rain",
4259.578368081543,
2646.7792890031856,
1,
360,
"Security Delay",
"VOD2DJ5",
11.916081066779922,
714.96486400679532,
"El Dorado International Airport",
"BOG",
"Bogota",
"CO",
{
"lat": "4.70159",
"lon": "-74.1469"
},
"CO-CUN",
"Thunder & Lightning",
6,
"2018-02-11T12:02:49Z"
],
[
530.7993561801253,
0,
"Kibana Airlines",
"Montreal / Pierre Elliott Trudeau International Airport",
"YUL",
"Montreal",
"CA",
{
"lat": "45.47060013",
"lon": "-73.74079895"
},
"CA-QC",
"Cloudy",
5261.14701710131,
3269.125194552134,
0,
0,
"No Delay",
"TQVZA43",
4.6150412430713237,
276.90247458427945,
"London Gatwick Airport",
"LGW",
"London",
"GB",
{
"lat": "51.14810181",
"lon": "-0.190277994"
},
"GB-ENG",
"Rain",
6,
"2018-02-11T11:45:58Z"
],
[
603.16832447941374,
0,
"Logstash Airways",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Clear",
11937.503792673972,
7417.6209639915214,
1,
15,
"Late Aircraft Delay",
"QYKRG4Z",
9.7242093592650569,
583.45256155590346,
"Comodoro Arturo Merino Benitez International Airport",
"SCL",
"Santiago",
"CL",
{
"lat": "-33.39300156",
"lon": "-70.78579712"
},
"SE-BD",
"Clear",
6,
"2018-02-11T11:38:46Z"
],
[
1194.0236452922072,
0,
"ES-Air",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Rain",
11778.102983821767,
7318.5738933514322,
0,
0,
"No Delay",
"18ALHUO",
11.547159788060556,
692.82958728363337,
"Tulsa International Airport",
"TUL",
"Tulsa",
"US",
{
"lat": "36.19839859",
"lon": "-95.88809967"
},
"US-OK",
"Hail",
6,
"2018-02-11T11:34:41Z"
],
[
852.28318962713251,
0,
"ES-Air",
"Shanghai Hongqiao International Airport",
"SHA",
"Shanghai",
"CN",
{
"lat": "31.19790077",
"lon": "121.3359985"
},
"SE-BD",
"Damaging Wind",
11741.628437197684,
7295.9096608293084,
1,
255,
"Carrier Delay",
"XHBN6H7",
19.303369791279081,
1158.2021874767449,
"OR Tambo International Airport",
"JNB",
"Johannesburg",
"ZA",
{
"lat": "-26.1392",
"lon": "28.246"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T11:27:47Z"
],
[
833.04593777381024,
0,
"JetBeats",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Rain",
7136.8453359577352,
4434.6300952175143,
1,
225,
"Carrier Delay",
"LYSOOZK",
11.184213891622642,
671.05283349735851,
"Jorge Chavez International Airport",
"LIM",
"Lima",
"PE",
{
"lat": "-12.0219",
"lon": "-77.114304"
},
"SE-BD",
"Hail",
6,
"2018-02-11T11:24:57Z"
],
[
630.77952610274542,
0,
"JetBeats",
"Helsinki Vantaa Airport",
"HEL",
"Helsinki",
"FI",
{
"lat": "60.31719971",
"lon": "24.9633007"
},
"FI-ES",
"Sunny",
6324.57895138743,
3929.9111634227543,
0,
0,
"No Delay",
"AMNDOW7",
7.5292606564136069,
451.75563938481639,
"Beijing Capital International Airport",
"PEK",
"Beijing",
"CN",
{
"lat": "40.08010101",
"lon": "116.5849991"
},
"SE-BD",
"Cloudy",
6,
"2018-02-11T11:23:23Z"
],
[
531.00009263065363,
0,
"ES-Air",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Rain",
291.90172192067166,
181.37932096597845,
1,
30,
"Security Delay",
"BO1KI4Z",
0.743251434933893,
44.595086096033583,
"Verona Villafranca Airport",
"VR10",
"Verona",
"IT",
{
"lat": "45.395699",
"lon": "10.8885"
},
"IT-34",
"Sunny",
6,
"2018-02-11T11:20:18Z"
],
[
357.56284153137523,
1,
"Logstash Airways",
"Shanghai Pudong International Airport",
"PVG",
"Shanghai",
"CN",
{
"lat": "31.14340019",
"lon": "121.8050003"
},
"SE-BD",
"Thunder & Lightning",
0.0,
0.0,
0,
0,
"No Delay",
"UUJQFTA",
0.0,
0.0,
"Shanghai Pudong International Airport",
"PVG",
"Shanghai",
"CN",
{
"lat": "31.14340019",
"lon": "121.8050003"
},
"SE-BD",
"Thunder & Lightning",
6,
"2018-02-11T11:19:12Z"
],
[
756.88094073994046,
1,
"Kibana Airlines",
"Treviso-Sant'Angelo Airport",
"TV01",
"Treviso",
"IT",
{
"lat": "45.648399",
"lon": "12.1944"
},
"IT-34",
"Heavy Fog",
8700.3509028134686,
5406.14741336437,
0,
0,
"No Delay",
"425S3GJ",
8.5297557870720286,
511.7853472243217,
"Incheon International Airport",
"ICN",
"Seoul",
"KR",
{
"lat": "37.46910095",
"lon": "126.4509964"
},
"SE-BD",
"Thunder & Lightning",
6,
"2018-02-11T11:11:47Z"
],
[
1014.0527874787632,
0,
"Logstash Airways",
"Vienna International Airport",
"VIE",
"Vienna",
"AT",
{
"lat": "48.11029816",
"lon": "16.56970024"
},
"AT-9",
"Thunder & Lightning",
6481.1079209495856,
4027.1737558592727,
1,
150,
"Security Delay",
"QG5GBQP",
11.501538779096647,
690.0923267457988,
"Montreal / Pierre Elliott Trudeau International Airport",
"YUL",
"Montreal",
"CA",
{
"lat": "45.47060013",
"lon": "-73.74079895"
},
"CA-QC",
"Thunder & Lightning",
6,
"2018-02-11T10:56:31Z"
],
[
929.69613599614559,
0,
"ES-Air",
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Clear",
9595.42771414762,
5962.3223587670618,
0,
0,
"No Delay",
"0Y830VQ",
9.995237202237103,
599.71423213422622,
"Tokyo Haneda International Airport",
"HND",
"Tokyo",
"JP",
{
"lat": "35.552299",
"lon": "139.779999"
},
"SE-BD",
"Rain",
6,
"2018-02-11T10:54:41Z"
],
[
780.86071688336551,
0,
"Logstash Airways",
"Narita International Airport",
"NRT",
"Tokyo",
"JP",
{
"lat": "35.76470184",
"lon": "140.3860016"
},
"SE-BD",
"Cloudy",
9733.5659568233386,
6048.157483312044,
0,
0,
"No Delay",
"YAC758U",
8.1113049640194479,
486.67829784116691,
"Charles de Gaulle International Airport",
"CDG",
"Paris",
"FR",
{
"lat": "49.01279831",
"lon": "2.549999952"
},
"FR-J",
"Clear",
6,
"2018-02-11T10:54:28Z"
],
[
1073.1833398797767,
0,
"ES-Air",
"Stockholm-Arlanda Airport",
"ARN",
"Stockholm",
"SE",
{
"lat": "59.65190125",
"lon": "17.91860008"
},
"SE-AB",
"Rain",
8547.0806747381539,
5310.9097090107234,
1,
45,
"NAS Delay",
"YNP23HR",
8.2474391883668012,
494.84635130200809,
"San Antonio International Airport",
"SAT",
"San Antonio",
"US",
{
"lat": "29.53370094",
"lon": "-98.46980286"
},
"US-TX",
"Rain",
6,
"2018-02-11T10:41:40Z"
],
[
752.17292470467737,
0,
"Logstash Airways",
"Shanghai Hongqiao International Airport",
"SHA",
"Shanghai",
"CN",
{
"lat": "31.19790077",
"lon": "121.3359985"
},
"SE-BD",
"Rain",
9154.0721553726125,
5688.0767290104613,
0,
0,
"No Delay",
"WK5II29",
7.62839346281051,
457.7036077686306,
"Leonardo da Vinci___Fiumicino Airport",
"RM11",
"Rome",
"IT",
{
"lat": "41.8002778",
"lon": "12.2388889"
},
"IT-62",
"Clear",
6,
"2018-02-11T10:41:40Z"
],
[
912.69191284222757,
0,
"Kibana Airlines",
"Stockholm-Arlanda Airport",
"ARN",
"Stockholm",
"SE",
{
"lat": "59.65190125",
"lon": "17.91860008"
},
"SE-AB",
"Clear",
8082.1718434091072,
5022.0287542061278,
0,
0,
"No Delay",
"8AXKQAR",
6.7351432028409226,
404.10859217045538,
"Itami Airport",
"ITM",
"Osaka",
"JP",
{
"lat": "34.78549957",
"lon": "135.4380035"
},
"SE-BD",
"Damaging Wind",
6,
"2018-02-11T10:36:27Z"
],
[
376.52968733115756,
0,
"Logstash Airways",
"New Chitose Airport",
"CTS",
"Chitose / Tomakomai",
"JP",
{
"lat": "42.77519989",
"lon": "141.6920013"
},
"SE-BD",
"Rain",
786.39909247766855,
488.64574166720632,
0,
0,
"No Delay",
"J8ENXN0",
1.0922209617745398,
65.533257706472384,
"Narita International Airport",
"NRT",
"Tokyo",
"JP",
{
"lat": "35.76470184",
"lon": "140.3860016"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T10:36:27Z"
],
[
825.74385646559858,
0,
"JetBeats",
"Warsaw Chopin Airport",
"WAW",
"Warsaw",
"PL",
{
"lat": "52.16569901",
"lon": "20.96710014"
},
"PL-MZ",
"Heavy Fog",
8727.4518136915485,
5422.9871386673994,
1,
345,
"Weather Delay",
"FWA54GG",
13.022876511409624,
781.3725906845774,
"Louis Armstrong New Orleans International Airport",
"MSY",
"New Orleans",
"US",
{
"lat": "29.99340057",
"lon": "-90.25800323"
},
"US-LA",
"Heavy Fog",
6,
"2018-02-11T10:29:03Z"
],
[
1181.8748577991928,
0,
"JetBeats",
"Tokyo Haneda International Airport",
"HND",
"Tokyo",
"JP",
{
"lat": "35.552299",
"lon": "139.779999"
},
"SE-BD",
"Clear",
11074.402963368044,
6881.3149726646652,
0,
0,
"No Delay",
"VQ493AH",
15.381115226900061,
922.86691361400369,
"Piedmont Triad International Airport",
"GSO",
"Greensboro",
"US",
{
"lat": "36.09780121",
"lon": "-79.93730164"
},
"US-NC",
"Rain",
6,
"2018-02-11T10:29:03Z"
],
[
729.78817067984289,
1,
"ES-Air",
"Vienna International Airport",
"VIE",
"Vienna",
"AT",
{
"lat": "48.11029816",
"lon": "16.56970024"
},
"AT-9",
"Rain",
7240.2829104873626,
4498.9032242251269,
1,
135,
"NAS Delay",
"FPS5ZUQ",
11.532413987804311,
691.94483926825865,
"Ukrainka Air Base",
"XHBU",
"Belogorsk",
"RU",
{
"lat": "51.169997",
"lon": "128.445007"
},
"RU-AMU",
"Damaging Wind",
6,
"2018-02-11T10:24:42Z"
],
[
611.37023245183173,
0,
"Logstash Airways",
"Jorge Chavez International Airport",
"LIM",
"Lima",
"PE",
{
"lat": "-12.0219",
"lon": "-77.114304"
},
"SE-BD",
"Sunny",
6777.5106171739053,
4211.3498525945388,
0,
0,
"No Delay",
"EG26UYG",
5.647925514311587,
338.87553085869524,
"Casper-Natrona County International Airport",
"CPR",
"Casper",
"US",
{
"lat": "42.90800095",
"lon": "-106.4639969"
},
"US-WY",
"Rain",
6,
"2018-02-11T10:24:30Z"
],
[
445.71492041764446,
0,
"JetBeats",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Sunny",
0.0,
0.0,
1,
180,
"Security Delay",
"XUAY4SM",
3.0,
180.0,
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Cloudy",
6,
"2018-02-11T10:21:53Z"
],
[
662.30699246014683,
0,
"ES-Air",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Heavy Fog",
10131.225294849231,
6295.2515402855015,
1,
240,
"Late Aircraft Delay",
"YWT6SEF",
13.932573818479638,
835.9544291087783,
"Ministro Pistarini International Airport",
"EZE",
"Buenos Aires",
"AR",
{
"lat": "-34.8222",
"lon": "-58.5358"
},
"AR-B",
"Sunny",
6,
"2018-02-11T10:13:32Z"
],
[
562.34461127886789,
0,
"JetBeats",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Rain",
2442.8775632135025,
1517.933743943807,
0,
0,
"No Delay",
"IYNT0MX",
1.9387917168361133,
116.32750301016679,
"Ukrainka Air Base",
"XHBU",
"Belogorsk",
"RU",
{
"lat": "51.169997",
"lon": "128.445007"
},
"RU-AMU",
"Thunder & Lightning",
6,
"2018-02-11T10:05:47Z"
],
[
739.13216536621019,
0,
"Logstash Airways",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Rain",
10316.155866415009,
6410.16207002046,
0,
0,
"No Delay",
"HU0ILVE",
9.04925953194299,
542.9555719165794,
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Hail",
6,
"2018-02-11T10:02:21Z"
],
[
802.57533633139542,
0,
"Logstash Airways",
"Tucson International Airport",
"TUS",
"Tucson",
"US",
{
"lat": "32.11610031",
"lon": "-110.9410019"
},
"US-AZ",
"Rain",
11077.248476805155,
6883.08309274161,
0,
0,
"No Delay",
"GNF75KM",
16.783709813341144,
1007.0225888004686,
"Shanghai Pudong International Airport",
"PVG",
"Shanghai",
"CN",
{
"lat": "31.14340019",
"lon": "121.8050003"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T09:52:27Z"
],
[
962.34881335032071,
0,
"JetBeats",
"Ministro Pistarini International Airport",
"EZE",
"Buenos Aires",
"AR",
{
"lat": "-34.8222",
"lon": "-58.5358"
},
"SE-BD",
"Damaging Wind",
11479.139700433523,
7132.8067215172905,
0,
0,
"No Delay",
"03OIL3V",
11.254058529836787,
675.24351179020721,
"Frankfurt am Main Airport",
"FRA",
"Frankfurt am Main",
"DE",
{
"lat": "50.033333",
"lon": "8.570556"
},
"DE-HE",
"Cloudy",
6,
"2018-02-11T09:16:17Z"
],
[
595.96128497553855,
0,
"JetBeats",
"Ottawa Macdonald-Cartier International Airport",
"YOW",
"Ottawa",
"CA",
{
"lat": "45.32249832",
"lon": "-75.66919708"
},
"CA-ON",
"Clear",
6002.0733859587053,
3729.5154957291325,
0,
0,
"No Delay",
"U2E5Q2L",
6.2521597770403181,
375.12958662241908,
"Frankfurt am Main Airport",
"FRA",
"Frankfurt am Main",
"DE",
{
"lat": "50.033333",
"lon": "8.570556"
},
"DE-HE",
"Clear",
6,
"2018-02-11T09:02:07Z"
],
[
685.72618108188817,
0,
"ES-Air",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Sunny",
7426.3681019075239,
4614.5312014755846,
0,
0,
"No Delay",
"0XG3GDP",
8.8409144070327663,
530.454864421966,
"Treviso-Sant'Angelo Airport",
"TV01",
"Treviso",
"IT",
{
"lat": "45.648399",
"lon": "12.1944"
},
"IT-34",
"Hail",
6,
"2018-02-11T08:58:52Z"
],
[
740.02837154345025,
0,
"Logstash Airways",
"Tokyo Haneda International Airport",
"HND",
"Tokyo",
"JP",
{
"lat": "35.552299",
"lon": "139.779999"
},
"SE-BD",
"Cloudy",
9371.8977264680834,
5823.4272638218326,
0,
0,
"No Delay",
"PTFUSLG",
10.413219696075649,
624.79318176453887,
"Cologne Bonn Airport",
"CGN",
"Cologne",
"DE",
{
"lat": "50.86589813",
"lon": "7.142739773"
},
"DE-NW",
"Heavy Fog",
6,
"2018-02-11T08:52:06Z"
],
[
623.32247720925147,
0,
"JetBeats",
"Ukrainka Air Base",
"XHBU",
"Belogorsk",
"RU",
{
"lat": "51.169997",
"lon": "128.445007"
},
"RU-AMU",
"Sunny",
9343.8857488189551,
5806.0214278730682,
0,
0,
"No Delay",
"YOQ53TS",
8.6517460637212551,
519.10476382327533,
"Portland International Jetport Airport",
"PWM",
"Portland",
"US",
{
"lat": "43.64619827",
"lon": "-70.30930328"
},
"US-ME",
"Hail",
6,
"2018-02-11T08:35:17Z"
],
[
1102.8144645388557,
0,
"JetBeats",
"Ministro Pistarini International Airport",
"EZE",
"Buenos Aires",
"AR",
{
"lat": "-34.8222",
"lon": "-58.5358"
},
"SE-BD",
"Hail",
18748.859646655459,
11650.001271732743,
1,
135,
"Carrier Delay",
"66FQQMD",
28.290082842577025,
1697.4049705546215,
"Itami Airport",
"ITM",
"Osaka",
"JP",
{
"lat": "34.78549957",
"lon": "135.4380035"
},
"SE-BD",
"Hail",
6,
"2018-02-11T08:28:21Z"
],
[
1157.5077271483356,
0,
"Kibana Airlines",
"Turin Airport",
"TO11",
"Torino",
"IT",
{
"lat": "45.200802",
"lon": "7.64963"
},
"IT-21",
"Heavy Fog",
16681.281547315732,
10365.267803102215,
0,
0,
"No Delay",
"1L71ZCH",
15.44563106232938,
926.73786373976282,
"Sydney Kingsford Smith International Airport",
"SYD",
"Sydney",
"AU",
{
"lat": "-33.94609833",
"lon": "151.177002"
},
"SE-BD",
"Damaging Wind",
6,
"2018-02-11T08:25:54Z"
],
[
697.2083968654315,
0,
"Logstash Airways",
"Manchester Airport",
"MAN",
"Manchester",
"GB",
{
"lat": "53.35369873",
"lon": "-2.274950027"
},
"GB-ENG",
"Cloudy",
8806.708392890594,
5472.2348937769639,
0,
0,
"No Delay",
"6J0PQVN",
7.7251828007812229,
463.51096804687336,
"Gimpo International Airport",
"GMP",
"Seoul",
"KR",
{
"lat": "37.5583",
"lon": "126.791"
},
"SE-BD",
"Hail",
6,
"2018-02-11T08:13:59Z"
],
[
1002.2862000309508,
0,
"Kibana Airlines",
"Warsaw Chopin Airport",
"WAW",
"Warsaw",
"PL",
{
"lat": "52.16569901",
"lon": "20.96710014"
},
"PL-MZ",
"Clear",
967.5830353995043,
601.22822429480846,
0,
0,
"No Delay",
"VVEA7OW",
0.89591021796250392,
53.754613077750236,
"Treviso-Sant'Angelo Airport",
"TV01",
"Treviso",
"IT",
{
"lat": "45.648399",
"lon": "12.1944"
},
"IT-34",
"Rain",
6,
"2018-02-11T08:06:52Z"
],
[
701.07419678804422,
0,
"Logstash Airways",
"Jorge Chavez International Airport",
"LIM",
"Lima",
"PE",
{
"lat": "-12.0219",
"lon": "-77.114304"
},
"SE-BD",
"Cloudy",
9842.94883622534,
6116.1248534964179,
1,
240,
"NAS Delay",
"5IXRGVO",
16.619165174647872,
997.14991047887224,
"Dublin Airport",
"DUB",
"Dublin",
"IE",
{
"lat": "53.42129898",
"lon": "-6.270070076"
},
"IE-D",
"Clear",
6,
"2018-02-11T08:00:22Z"
],
[
1083.6016862775346,
0,
"Logstash Airways",
"Winnipeg / James Armstrong Richardson International Airport",
"YWG",
"Winnipeg",
"CA",
{
"lat": "49.90999985",
"lon": "-97.23989868"
},
"CA-MB",
"Sunny",
7446.4285318079565,
4626.99617471961,
0,
0,
"No Delay",
"P3UKWC7",
6.5319548524631195,
391.91729114778718,
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Cloudy",
6,
"2018-02-11T07:51:48Z"
],
[
766.96531716480865,
0,
"Kibana Airlines",
"Warsaw Chopin Airport",
"WAW",
"Warsaw",
"PL",
{
"lat": "52.16569901",
"lon": "20.96710014"
},
"PL-MZ",
"Rain",
8142.2197451414922,
5059.34079049693,
0,
0,
"No Delay",
"Y9U4Y6U",
6.7851831209512437,
407.11098725707461,
"Guangzhou Baiyun International Airport",
"CAN",
"Guangzhou",
"CN",
{
"lat": "23.39240074",
"lon": "113.2990036"
},
"SE-BD",
"Rain",
6,
"2018-02-11T07:28:30Z"
],
[
403.6147064445326,
0,
"ES-Air",
"Tulsa International Airport",
"TUL",
"Tulsa",
"US",
{
"lat": "36.19839859",
"lon": "-95.88809967"
},
"US-OK",
"Cloudy",
2186.5223553595033,
1358.6420028033181,
0,
0,
"No Delay",
"59G22E0",
2.6030028039994089,
156.18016823996453,
"Spokane International Airport",
"GEG",
"Spokane",
"US",
{
"lat": "47.61989975",
"lon": "-117.5339966"
},
"US-WA",
"Clear",
6,
"2018-02-11T07:14:34Z"
],
[
506.63232728025105,
0,
"Kibana Airlines",
"Rajiv Gandhi International Airport",
"HYD",
"Hyderabad",
"IN",
{
"lat": "17.23131752",
"lon": "78.42985535"
},
"SE-BD",
"Rain",
6474.6233011172881,
4023.144399902872,
1,
285,
"Late Aircraft Delay",
"STIRQ1F",
9.8885899215216568,
593.31539529129941,
"Olenya Air Base",
"XLMO",
"Olenegorsk",
"RU",
{
"lat": "68.15180206",
"lon": "33.46390152"
},
"RU-MUR",
"Sunny",
6,
"2018-02-11T07:01:23Z"
],
[
979.92516075176559,
0,
"ES-Air",
"Sheremetyevo International Airport",
"SVO",
"Moscow",
"RU",
{
"lat": "55.972599",
"lon": "37.4146"
},
"RU-MOS",
"Clear",
14410.643651074781,
8954.3588263757028,
1,
150,
"NAS Delay",
"MQTAG6N",
22.514782848714972,
1350.8869709228984,
"Melbourne International Airport",
"MEL",
"Melbourne",
"AU",
{
"lat": "-37.673302",
"lon": "144.843002"
},
"SE-BD",
"Thunder & Lightning",
6,
"2018-02-11T06:55:18Z"
],
[
1085.342629338636,
0,
"Kibana Airlines",
"Olenya Air Base",
"XLMO",
"Olenegorsk",
"RU",
{
"lat": "68.15180206",
"lon": "33.46390152"
},
"RU-MUR",
"Sunny",
8444.50682235057,
5247.173272060274,
1,
315,
"Carrier Delay",
"F0Q6RAJ",
11.951989541548072,
717.11937249288428,
"Los Angeles International Airport",
"LAX",
"Los Angeles",
"US",
{
"lat": "33.94250107",
"lon": "-118.4079971"
},
"US-CA",
"Thunder & Lightning",
6,
"2018-02-11T06:54:51Z"
],
[
673.41124567239672,
1,
"ES-Air",
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Hail",
787.48113960826424,
489.31809458280156,
0,
0,
"No Delay",
"P5DAUJO",
1.0937238050114782,
65.623428300688687,
"Berlin-Tegel Airport",
"TXL",
"Berlin",
"DE",
{
"lat": "52.5597",
"lon": "13.2877"
},
"DE-BE",
"Rain",
6,
"2018-02-11T06:50:34Z"
],
[
326.34911293769522,
0,
"Kibana Airlines",
"Treviso-Sant'Angelo Airport",
"TV01",
"Treviso",
"IT",
{
"lat": "45.648399",
"lon": "12.1944"
},
"IT-34",
"Cloudy",
427.56775974975113,
265.67828863794881,
0,
0,
"No Delay",
"QGD7L5C",
0.59384411076354326,
35.6306466458126,
"Leonardo da Vinci - Fiumicino Airport",
"FCO",
"Rome",
"IT",
{
"lat": "41.8002778",
"lon": "12.2388889"
},
"SE-BD",
"Heavy Fog",
6,
"2018-02-11T06:36:31Z"
],
[
841.68625617189434,
0,
"Logstash Airways",
"Ukrainka Air Base",
"XHBU",
"Belogorsk",
"RU",
{
"lat": "51.169997",
"lon": "128.445007"
},
"RU-AMU",
"Clear",
6232.2312582294189,
3872.5289672247936,
0,
0,
"No Delay",
"RT13KQS",
9.4427746336809371,
566.56647802085627,
"Stockholm-Arlanda Airport",
"ARN",
"Stockholm",
"SE",
{
"lat": "59.65190125",
"lon": "17.91860008"
},
"SE-AB",
"Sunny",
6,
"2018-02-11T06:31:57Z"
],
[
540.57678193712161,
0,
"JetBeats",
"Sydney Kingsford Smith International Airport",
"SYD",
"Sydney",
"AU",
{
"lat": "-33.94609833",
"lon": "151.177002"
},
"SE-BD",
"Rain",
15551.352695623931,
9663.16256538312,
0,
0,
"No Delay",
"NPLKNZD",
12.95946057968661,
777.5676347811966,
"Lester B. Pearson International Airport",
"YYZ",
"Toronto",
"CA",
{
"lat": "43.67720032",
"lon": "-79.63059998"
},
"CA-ON",
"Clear",
6,
"2018-02-11T06:29:14Z"
],
[
988.77060418789245,
0,
"ES-Air",
"Tulsa International Airport",
"TUL",
"Tulsa",
"US",
{
"lat": "36.19839859",
"lon": "-95.88809967"
},
"US-OK",
"Clear",
8663.017092577129,
5382.949259187053,
0,
0,
"No Delay",
"T07V2ZX",
9.023976138101176,
541.43856828607056,
"Cagliari Elmas Airport",
"CA07",
"Cagliari",
"IT",
{
"lat": "39.251499",
"lon": "9.05428"
},
"IT-88",
"Clear",
6,
"2018-02-11T06:24:02Z"
],
[
667.58314412471543,
0,
"JetBeats",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Sunny",
9076.9042045365732,
5640.12678739696,
0,
0,
"No Delay",
"3IJIMJ3",
7.9621966706461169,
477.731800238767,
"El Dorado International Airport",
"BOG",
"Bogota",
"CO",
{
"lat": "4.70159",
"lon": "-74.1469"
},
"CO-CUN",
"Heavy Fog",
6,
"2018-02-11T06:22:51Z"
],
[
983.42924366226,
1,
"Kibana Airlines",
"Venice Marco Polo Airport",
"VE05",
"Venice",
"IT",
{
"lat": "45.505299",
"lon": "12.3519"
},
"IT-34",
"Thunder & Lightning",
6232.7835257402567,
3872.8721303464372,
1,
270,
"NAS Delay",
"M6R2H4T",
11.425315028600286,
685.51890171601713,
"Chhatrapati Shivaji International Airport",
"BOM",
"Mumbai",
"IN",
{
"lat": "19.08869934",
"lon": "72.86789703"
},
"SE-BD",
"Thunder & Lightning",
6,
"2018-02-11T06:19:58Z"
],
[
1176.1905251500007,
0,
"ES-Air",
"Il Caravaggio International Airport",
"BG01",
"Bergamo",
"IT",
{
"lat": "45.673901",
"lon": "9.70417"
},
"IT-25",
"Sunny",
11913.334123112405,
7402.6026276000684,
0,
0,
"No Delay",
"FY4JKB9",
14.182540622752864,
850.95243736517182,
"Comodoro Arturo Merino Benitez International Airport",
"SCL",
"Santiago",
"CL",
{
"lat": "-33.39300156",
"lon": "-70.78579712"
},
"SE-BD",
"Clear",
6,
"2018-02-11T06:17:54Z"
],
[
479.3173585143557,
0,
"ES-Air",
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Sunny",
0.0,
0.0,
1,
330,
"Carrier Delay",
"RWJDQQH",
5.5,
330.0,
"Xi'an Xianyang International Airport",
"XIY",
"Xi'an",
"CN",
{
"lat": "34.447102",
"lon": "108.751999"
},
"SE-BD",
"Rain",
6,
"2018-02-11T06:04:44Z"
],
[
1181.7700219583969,
0,
"Kibana Airlines",
"Ottawa Macdonald-Cartier International Airport",
"YOW",
"Ottawa",
"CA",
{
"lat": "45.32249832",
"lon": "-75.66919708"
},
"CA-ON",
"Cloudy",
5648.0124842383129,
3509.5122511025065,
0,
0,
"No Delay",
"0S0S03D",
4.7066770701985945,
282.40062421191567,
"Amsterdam Airport Schiphol",
"AMS",
"Amsterdam",
"NL",
{
"lat": "52.30860138",
"lon": "4.76388979"
},
"NL-NH",
"Hail",
6,
"2018-02-11T06:01:48Z"
],
[
947.59293297909494,
0,
"Kibana Airlines",
"Zurich Airport",
"ZRH",
"Zurich",
"CH",
{
"lat": "47.464699",
"lon": "8.54917"
},
"CH-ZH",
"Clear",
9001.0705118200513,
5593.0059153419352,
0,
0,
"No Delay",
"O0CZ9AW",
9.37611511647922,
562.5669069887532,
"New Chitose Airport",
"CTS",
"Chitose / Tomakomai",
"JP",
{
"lat": "42.77519989",
"lon": "141.6920013"
},
"SE-BD",
"Heavy Fog",
6,
"2018-02-11T05:55:58Z"
],
[
514.788416546782,
0,
"JetBeats",
"Mariscal Sucre International Airport",
"UIO",
"Quito",
"EC",
{
"lat": "-0.129166667",
"lon": "-78.3575"
},
"SE-BD",
"Heavy Fog",
14714.628391255823,
9143.2461868039536,
0,
0,
"No Delay",
"O0UFN3A",
16.349587101395358,
980.97522608372151,
"Chubu Centrair International Airport",
"NGO",
"Tokoname",
"JP",
{
"lat": "34.85839844",
"lon": "136.8049927"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T05:43:48Z"
],
[
942.893674824945,
0,
"Logstash Airways",
"Luis Munoz Marin International Airport",
"SJU",
"San Juan",
"PR",
{
"lat": "18.43939972",
"lon": "-66.00180054"
},
"PR-U-A",
"Cloudy",
13663.716346166211,
8490.2397164100475,
1,
285,
"NAS Delay",
"EJSEIOZ",
16.735716093128257,
1004.1429655876954,
"Gimpo International Airport",
"GMP",
"Seoul",
"KR",
{
"lat": "37.5583",
"lon": "126.791"
},
"SE-BD",
"Sunny",
6,
"2018-02-11T05:43:23Z"
]
] azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/dataset.csv 0000664 0000000 0000000 00000003041 14172227630 0025550 0 ustar 00root root 0000000 0000000 0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null
1,00000001-0000-0000-0001-020304050607,1.0001,1.01,1,1,1,1,1,1,1,1,2015-01-01T01:01:01.0000000Z,One,"One",1,00:00:01.0010001,,"{""rowId"": 1, ""arr"": [0,1]}"
2,00000002-0000-0000-0001-020304050607,2.0002,2.02,0,2,2,2,2,2,2,2,2016-01-01T01:01:01.0000000Z,Two,"Two",2,-00:00:02.0020002,,"{""rowId"": 2, ""arr"": [0,2]}"
3,00000003-0000-0000-0001-020304050607,3.0003,3.03,1,3,3,3,3,3,3,3,2017-01-01T01:01:01.0000000Z,Three,"Three",3,00:00:03.0030003,,"{""rowId"": 3, ""arr"": [0,3]}"
4,00000004-0000-0000-0001-020304050607,4.0004,4.04,0,4,4,4,4,4,4,4,2018-01-01T01:01:01.0000000Z,Four,"Four",4,-00:00:04.0040004,,"{""rowId"": 4, ""arr"": [0,4]}"
5,00000005-0000-0000-0001-020304050607,5.0005,5.05,1,5,5,5,5,5,5,5,2019-01-01T01:01:01.0000000Z,Five,"Five",5,00:00:05.0050005,,"{""rowId"": 5, ""arr"": [0,5]}"
6,00000006-0000-0000-0001-020304050607,6.0006,6.06,0,6,6,6,6,6,6,6,2020-01-01T01:01:01.0000000Z,Six,"Six",6,-00:00:06.0060006,,"{""rowId"": 6, ""arr"": [0,6]}"
7,00000007-0000-0000-0001-020304050607,7.0007,7.07,1,7,7,7,7,7,7,7,2021-01-01T01:01:01.0000000Z,Seven,"Seven",7,00:00:07.0070007,,"{""rowId"": 7, ""arr"": [0,7]}"
8,00000008-0000-0000-0001-020304050607,8.0008,8.08,0,8,8,8,8,8,8,8,2022-01-01T01:01:01.0000000Z,Eight,"Eight",8,-00:00:08.0080008,,"{""rowId"": 8, ""arr"": [0,8]}"
9,00000009-0000-0000-0001-020304050607,9.0009,9.09,1,9,9,9,9,9,9,9,2023-01-01T01:01:01.0000000Z,Nine,"Nine",9,00:00:09.0090009,,"{""rowId"": 9, ""arr"": [0,9]}" azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/dataset.csv.gz 0000664 0000000 0000000 00000000715 14172227630 0026174 0 ustar 00root root 0000000 0000000 ‹ó@Zÿdataset.csv }ÔËj[A à}ž"h-—¹_¼O!›vѬRº(ôЂ
‡æ¥ï^ýƒµPéŒeFÏùõ!° »ëë`pÁE—\vÅUvÿ©à|:à¢pþ8Þ ®Yü¸í&œ„Ëî8ÞÌç—çç¯CýrèˆóhrüS2oLr{ oýˆcúM´_Þî o=ß}ßw|þêØûC7AyaÉ Ð¶aËù2å=¼Éjä ¹x¸úFžõë ðEõÅ¥/bbD‹²‡hJ|uî{Ú7YàhÄQWˆ¼8"-1Zb1)1-‰ Cš<Àɔ۔øñò²3á$¹ªKD\‰V˜¬0A˜U˜—ÂŒ™-˳)ö¹ðô*;ÄIœu…HË#гf ‹ËX0² É\L¹9~9½3ÉArQˆ°2ò¬¯X_¯ª¯.}G“ 4WSâósßöºEˆF\u…È«#Ò«%V›Û’Ø0´¡ÉÜL 1L‰w§ŸO¿˜F#¹¬kD`™Öج±ÁØÕØ—ÆŽ© Ë»)1Æ©ñÓ †8‰».i}Z`·À.À¿Jzå! azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/dataset.csv.zip 0000664 0000000 0000000 00000002163 14172227630 0026355 0 ustar 00root root 0000000 0000000 PK Õf.PJzå² ! dataset.csv}ÔËJ,1à½O!µ®irOgöç€]èJq!Øè€Ì@ã
Äw·þ¶`2Õ¤¦úÏG-F±Z?»âÐ;e”UNyTdõG¥ÝƒúFéýò kÖ-ßNó‰ 'aXí—‡ùøöòr¦ó¥º{駥ÉQ.÷Í˯Ž“ $ƒË½¿¿ê%Žé‹h>}\<íÏ5Ÿ=Ì3¾ß)Ö÷ßtf2ÏtyfÀšÁ6¶^hòn>d5r îV  ò*Ÿ)}>›}¶ë³FЬìÁn ¾Øö=Ï“,pi$Ã+y‘Ñ–D ¢ËD×%º#hò»m86‰ÿOo3N’ѼDÄ9$VBW
„> }WèŒ yY¢ß„©-<¼Ëq’Œ®@¤yV@_=€!CŒ É Š22Ù^ >™ä Ì DX@^å ¥/À³/v}Q"[”ÆmÁ§Û¾é}:2-dx%"/"²"Æ’A 3qìÇ#hòÛÑ4‰ÿOϯLK#ÎkDàˆÌÊ8–ÆÆ”©kLFÐ’¬1m FÛ4^ ðgˆ“dt"-!°¦˜øPK Õf.PJzå¯ ! more_csv/dataset2.csv}ÔËj[A à}ž"h-—¹_¼O!›vѬRº(ôЂ
‡æ¥ï^ýƒµPéŒeFÏùõ!° »ëë`pÁE—\vÅUvÿ©à|:à¢pþ8Þ ®Yü¸í&œ„Ëî8ÞÌç—çç¯CýrèˆóhrüS2oLr{ oýˆcúM´_Þî o=ß}ßw|þêØûC7AyaÉ Ð¶aËù2å=¼Éjä ¹x¸úFžõë ðEõÅ¥/bbD‹²‡hJ|uî{Ú7YàhÄQWˆ¼8"-1Zb1)1-‰ Cš<Àɔ۔øñò²3á$¹ªKD\‰V˜¬0A˜U˜—ÂŒ™-˳)ö¹ðô*;ÄIœu…HË#гf ‹ËX0² É\L¹9~9½3ÉArQˆ°2ò¬¯X_¯ª¯.}G“ 4WSâósßöºEˆF\u…È«#Ò«%V›Û’Ø0´¡ÉÜL 1L‰w§ŸO¿˜F#¹¬kD`™Öج±ÁØÕØ—ÆŽ© Ë»)1Æ©ñÓ †8‰».i}Z`·À.À¿PK Õf.PJzå² ! $ dataset.csv
ukh¤ÀÊÕukh¤ÀÊÕukh¤ÀÊÕPK Õf.PJzå¯ ! Û more_csv/dataset2.csvPK ½ azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/dataset.json 0000664 0000000 0000000 00000001407 14172227630 0025732 0 ustar 00root root 0000000 0000000 {"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}
{"rownumber": 1, "rowguid": "00000001-0000-0000-0001-020304050607", "xdouble": 1.00001, "xfloat": 1.01, "xbool": 1, "xint16": 1, "xint32": 1, "xint64": 1, "xuint8": 1, "xuint16": 1, "xuint32": 1, "xuint64": 1, "xdate": "2015-01-01T01:01:01Z", "xsmalltext": "One", "xtext": "One", "xnumberAsText": "1", "xtime": "00:00:01.0010001", "xtextWithNulls": null, "xdynamicWithNulls": "{\"rowId\":1,\"arr\":[0,1]}"} azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/dataset.jsonz.gz 0000664 0000000 0000000 00000000464 14172227630 0026545 0 ustar 00root root 0000000 0000000 ‹4íµV dataset.json_ •Akƒ0Çïƒ}‡’³–÷¬uÅÛŽ»l— sÙˆ h Åï>mLÖÂV êï½üÃËïHZù%tSÑ–ä+ˆV†?4«G"0?qðÂØ@
[ÈàŽŒ‘¾–ºâÔ °Ãï\–jÁJJ>ŸÞ3¡0ó`“x¥'Ђ©ƒ)´´lj¡%V—ŠšÉÀ46“â 0·ë`'íš’sE{ev h+mõ¬0¹ïös¦m¬¡“—Ü.—}fêóQsÞ]1~í(ߢlØ›ß"d¸½ ãEãø㸶g}*¡§ }íèkG_;žLï ! ¤´‹ýÒ¾ý[û“ ¶ò¹t¼$ÝÜͯ–,Œé‡º 9F)Ûvü{_2ü +}á( azure-kusto-python-3.0.1/azure-kusto-ingest/tests/input/dataset.tsv 0000664 0000000 0000000 00000003042 14172227630 0025572 0 ustar 00root root 0000000 0000000 0 00000000-0000-0000-0001-020304050607 0 0 0 0 0 0 0 0 0 0 2014-01-01T01:01:01.0000000Z Zero "Zero" 0 00:00:00 null
1 00000001-0000-0000-0001-020304050607 1.0001 1.01 1 1 1 1 1 1 1 1 2015-01-01T01:01:01.0000000Z One "One" 1 00:00:01.0010001 "{""rowId"": 1, ""arr"": [0,1]}"
2 00000002-0000-0000-0001-020304050607 2.0002 2.02 0 22 2 2 2 2 2 2 2016-01-01T01:01:01.0000000Z Two "Two" 2 -00:00:02.0020002 "{""rowId"": 2, ""arr"": [0,2]}"
3 00000003-0000-0000-0001-020304050607 3.0003 3.03 1 3 3 3 3 3 3 3 2017-01-01T01:01:01.0000000Z Three "Three" 3 00:00:03.0030003 "{""rowId"": 3, ""arr"": [0,3]}"
4 00000004-0000-0000-0001-020304050607 4.0004 4.04 0 4 4 4 4 4 4 4 2018-01-01T01:01:01.0000000Z Four "Four" 4 -00:00:04.0040004 "{""rowId"": 4, ""arr"": [0,4]}"
5 00000005-0000-0000-0001-020304050607 5.0005 5.05 1 5 5 5 5 5 5 5 2019-01-01T01:01:01.0000000Z Five "Five" 5 00:00:05.0050005 "{""rowId"": 5, ""arr"": [0,5]}"
6 00000006-0000-0000-0001-020304050607 6.0006 6.06 0 6 6 6 6 6 6 6 2020-01-01T01:01:01.0000000Z Six "Six" 6 -00:00:06.0060006 "{""rowId"": 6, ""arr"": [0,6]}"
7 00000007-0000-0000-0001-020304050607 7.0007 7.07 1 7 7 7 7 7 7 7 2021-01-01T01:01:01.0000000Z Seven "Seven" 7 00:00:07.0070007 "{""rowId"": 7, ""arr"": [0,7]}"
8 00000008-0000-0000-0001-020304050607 8.0008 8.08 0 8 8 8 8 8 8 8 2022-01-01T01:01:01.0000000Z Eight "Eight" 8 -00:00:08.0080008 "{""rowId"": 8, ""arr"": [0,8]}"
9 00000009-0000-0000-0001-020304050607 9.0009 9.09 1 9 9 9 9 9 9 9 2023-01-01T01:01:01.0000000Z Nine "Nine" 9 00:00:09.0090009 "{""rowId"": 9, ""arr"": [0,9]}" azure-kusto-python-3.0.1/azure-kusto-ingest/tests/sample.py 0000664 0000000 0000000 00000022734 14172227630 0024114 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import io
from azure.kusto.data import KustoConnectionStringBuilder
from azure.kusto.data.data_format import DataFormat
from azure.kusto.ingest import (
QueuedIngestClient,
IngestionProperties,
FileDescriptor,
BlobDescriptor,
StreamDescriptor,
KustoStreamingIngestClient,
ManagedStreamingIngestClient,
IngestionStatus,
)
##################################################################
## AUTH ##
##################################################################
cluster = "https://ingest-{cluster_name}.kusto.windows.net/"
# In case you want to authenticate with AAD application.
client_id = ""
client_secret = ""
# read more at https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id
authority_id = ""
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, client_id, client_secret, authority_id)
# In case you want to authenticate with AAD application certificate.
filename = "path to a PEM certificate"
with open(filename, "r") as pem_file:
PEM = pem_file.read()
thumbprint = "certificate's thumbprint"
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(cluster, client_id, PEM, thumbprint, authority_id)
# In case you want to authenticate with AAD application certificate Subject Name & Issuer
filename = "path to a PEM certificate"
with open(filename, "r") as pem_file:
PEM = pem_file.read()
filename = "path to a public certificate"
with open(filename, "r") as cert_file:
public_certificate = cert_file.read()
thumbprint = "certificate's thumbprint"
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_sni_authentication(cluster, client_id, PEM, public_certificate, thumbprint, authority_id)
# In case you want to authenticate with a System Assigned Managed Service Identity (MSI)
kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster)
# In case you want to authenticate with a User Assigned Managed Service Identity (MSI)
user_assigned_client_id = "the AAD identity client id"
kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster, client_id=user_assigned_client_id)
# In case you want to authenticate with AAD username and password
username = ""
password = ""
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(cluster, username, password, authority_id)
# In case you want to authenticate with AAD device code.
# Please note that if you choose this option, you'll need to autenticate for every new instance that is initialized.
# It is highly recommended to create one instance and use it for all of your queries.
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster)
# The authentication method will be taken from the chosen KustoConnectionStringBuilder.
client = QueuedIngestClient(kcsb)
# there are more options for authenticating - see azure-kusto-data samples
##################################################################
## INGESTION ##
##################################################################
# there are a lot of useful properties, make sure to go over docs and check them out
ingestion_props = IngestionProperties(
database="{database_name}",
table="{table_name}",
data_format=DataFormat.CSV,
# in case status update for success are also required
# report_level=ReportLevel.FailuresAndSuccesses,
# in case a mapping is required
# ingestion_mapping_reference="{json_mapping_that_already_exists_on_table}"
# ingestion_mapping_type=IngestionMappingType.JSON
)
# ingest from file
file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes.
client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_props)
result = client.ingest_from_file("{filename}.csv", ingestion_properties=ingestion_props)
# Inspect the result for useful information, such as source_id and blob_url
print(repr(result))
# ingest from blob
blob_descriptor = BlobDescriptor(
"https://{path_to_blob}.csv.gz?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
10,
) # 10 is the raw size of the data in bytes.
client.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_props)
# ingest from dataframe
import pandas
fields = ["id", "name", "value"]
rows = [[1, "abc", 15.3], [2, "cde", 99.9]]
df = pandas.DataFrame(data=rows, columns=fields)
client.ingest_from_dataframe(df, ingestion_properties=ingestion_props)
# ingest a whole folder.
import os
path = "folder/path"
[client.ingest_from_file(f, ingestion_properties=ingestion_props) for f in os.listdir(path)]
##################################################################
## INGESTION STATUS ##
##################################################################
# if status updates are required, something like this can be done
import pprint
import time
from azure.kusto.ingest.status import KustoIngestStatusQueues
qs = KustoIngestStatusQueues(client)
MAX_BACKOFF = 180
backoff = 1
while True:
################### NOTICE ####################
# in order to get success status updates,
# make sure ingestion properties set the
# reportLevel=ReportLevel.FailuresAndSuccesses.
if qs.success.is_empty() and qs.failure.is_empty():
time.sleep(backoff)
backoff = min(backoff * 2, MAX_BACKOFF)
print("No new messages. backing off for {} seconds".format(backoff))
continue
backoff = 1
success_messages = qs.success.pop(10)
failure_messages = qs.failure.pop(10)
pprint.pprint("SUCCESS : {}".format(success_messages))
pprint.pprint("FAILURE : {}".format(failure_messages))
# you can of course separate them and dump them into a file for follow up investigations
with open("successes.log", "w+") as sf:
for sm in success_messages:
sf.write(str(sm))
with open("failures.log", "w+") as ff:
for fm in failure_messages:
ff.write(str(fm))
##################################################################
## STREAMING INGEST ##
##################################################################
# Authenticate against this cluster endpoint as shows in the Auth section
cluster = "https://{cluster_name}.kusto.windows.net"
client = KustoStreamingIngestClient(kcsb)
ingestion_properties = IngestionProperties(database="{database_name}", table="{table_name}", data_format=DataFormat.CSV)
# ingest from file
file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes.
client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_properties)
client.ingest_from_file("{filename}.csv", ingestion_properties=ingestion_properties)
# ingest from dataframe
import pandas
fields = ["id", "name", "value"]
rows = [[1, "abc", 15.3], [2, "cde", 99.9]]
df = pandas.DataFrame(data=rows, columns=fields)
client.ingest_from_dataframe(df, ingestion_properties=ingestion_properties)
# ingest from stream
byte_sequence = b"56,56,56"
bytes_stream = io.BytesIO(byte_sequence)
client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
stream_descriptor = StreamDescriptor(bytes_stream)
client.ingest_from_stream(stream_descriptor, ingestion_properties=ingestion_properties)
str_sequence = u"57,57,57"
str_stream = io.StringIO(str_sequence)
client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
##################################################################
## NANAGED STREAMING INGEST ##
##################################################################
# Managed streaming ingest client will try to use streaming ingestion for performance, but will fall back to queued ingestion if unable.
dm_cluster = "https://ingest-{cluster_name}.kusto.windows.net"
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(dm_cluster, client_id, client_secret, authority_id)
# Create it from a dm connection string
client = ManagedStreamingIngestClient.from_dm_kcsb(kcsb)
# or an engine connection string, like a streaming ingestion client with `from_engine_kcsb`
# or provide both: `ManagedStreamingIngestClient(engine_kcsb, dm_kcsb)`
# use client as you would a streaming or queued ingestion client
byte_sequence = b"56,56,56"
bytes_stream = io.BytesIO(byte_sequence)
client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
ingestion_properties = IngestionProperties(database="{database_name}", table="{table_name}", data_format=DataFormat.CSV)
# ingest from file
file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes.
result = client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_properties)
# inspect the result to see what type of ingestion was preformed
if result.status == IngestionStatus.QUEUED:
# fell back to queued ingestion
pass
# Managed streaming ingest client will fall back to queued if:
# - Multiple transient errors were encountered when trying to do streaming ingestion
# - The ingestion is too large for streaming ingestion (over 4MB)
# - The ingestion is directly for a blob
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/test_connection_string.py 0000664 0000000 0000000 00000003131 14172227630 0027405 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import unittest
from azure.kusto.ingest._resource_manager import _ResourceUri
class ResourceUriTests(unittest.TestCase):
"""Tests class connection_string."""
def test_blob_uri(self):
"""Tests parsing blob uris."""
storage_name = "storageaccountname"
container_name = "containername"
endpoint_suffix = "core.windows.net"
container_sas = "somesas"
uri = "https://{}.blob.{}/{}?{}".format(storage_name, endpoint_suffix, container_name, container_sas)
connection_string = _ResourceUri.parse(uri)
assert connection_string.storage_account_name == storage_name
assert connection_string.object_type == "blob"
assert connection_string.endpoint_suffix == endpoint_suffix
assert connection_string.sas == container_sas
assert connection_string.object_name == container_name
def test_queue_uri(self):
"""Tests parsing queues uris."""
storage_name = "storageaccountname"
queue_name = "queuename"
endpoint_suffix = "core.windows.net"
queue_sas = "somesas"
uri = "https://{}.queue.{}/{}?{}".format(storage_name, endpoint_suffix, queue_name, queue_sas)
connection_string = _ResourceUri.parse(uri)
assert connection_string.storage_account_name == storage_name
assert connection_string.object_type == "queue"
assert connection_string.endpoint_suffix == endpoint_suffix
assert connection_string.sas == queue_sas
assert connection_string.object_name == queue_name
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/test_descriptors.py 0000664 0000000 0000000 00000014672 14172227630 0026235 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import sys
import uuid
from io import BytesIO
from os import path
import pytest
from azure.kusto.ingest import FileDescriptor, BlobDescriptor, StreamDescriptor
class TestDescriptors:
"""Test class for FileDescriptor and BlobDescriptor."""
# this is the size with LF line endings
uncompressed_size = 1569
# this is the size with CRLF line endings
uncompressed_size_2 = 1578
mock_size = 10
INVALID_UUID = "12345"
TEST_UUID_STR = "5bcc12b7-e35c-4c76-a40a-2d89e6c2c7dd"
TEST_UUID = uuid.UUID("5bcc12b7-e35c-4c76-a40a-2d89e6c2c7dd", version=4)
def test_unzipped_file_with_size(self):
"""Tests FileDescriptor with size and unzipped file."""
filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv")
descriptor = FileDescriptor(filePath, self.mock_size)
with descriptor.open(True) as stream:
assert descriptor.size == self.mock_size
assert descriptor.stream_name.endswith(".csv.gz")
if sys.version_info[0] >= 3:
assert stream.readable()
assert stream.tell() == 0
assert stream.closed is True
def test_unzipped_file_without_size(self):
"""Tests FileDescriptor without size and unzipped file."""
filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv")
descriptor = FileDescriptor(filePath, 0)
with descriptor.open(True) as stream:
# TODO: since we don't know if the file is opened on CRLF system or an LF system, allow both sizes
# a more robust approach would be to open the file and check
assert descriptor.size in (self.uncompressed_size, self.uncompressed_size_2)
assert descriptor.stream_name.endswith(".csv.gz")
if sys.version_info[0] >= 3:
assert stream.readable()
assert stream.tell() == 0
assert stream.closed is True
def test_zipped_file_with_size(self):
"""Tests FileDescriptor with size and zipped file."""
filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv.gz")
descriptor = FileDescriptor(filePath, self.mock_size)
with descriptor.open(False) as stream:
assert descriptor.size == self.mock_size
assert descriptor.stream_name.endswith(".csv.gz")
if sys.version_info[0] >= 3:
assert stream.readable()
assert stream.tell() == 0
assert stream.closed is True
def test_gzip_file_without_size(self):
"""Tests FileDescriptor without size and zipped file."""
filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv.gz")
descriptor = FileDescriptor(filePath, 0)
with descriptor.open(False) as stream:
assert descriptor.size == self.uncompressed_size
assert descriptor.stream_name.endswith(".csv.gz")
if sys.version_info[0] >= 3:
assert stream.readable()
assert stream.tell() == 0
assert stream.closed is True
def test_zip_file_without_size(self):
"""Tests FileDescriptor without size and zipped file."""
filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv.zip")
descriptor = FileDescriptor(filePath, 0)
with descriptor.open(False) as stream:
# the zip archive contains 2 copies of the source file
assert descriptor.size == self.uncompressed_size * 2
assert descriptor.stream_name.endswith(".csv.zip")
if sys.version_info[0] >= 3:
assert stream.readable()
assert stream.tell() == 0
assert stream.closed is True
def test_unzipped_file_dont_compress(self):
"""Tests FileDescriptor with size and unzipped file."""
filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv")
descriptor = FileDescriptor(filePath, self.mock_size)
with descriptor.open(False) as stream:
assert descriptor.size == self.mock_size
assert descriptor.stream_name.endswith(".csv")
if sys.version_info[0] >= 3:
assert stream.readable()
assert stream.tell() == 0
assert stream.closed is True
def test_uuid_stream_descriptor(self):
dummy_stream = BytesIO(b"dummy")
descriptor = StreamDescriptor(dummy_stream)
assert descriptor.source_id
assert descriptor.source_id != TestDescriptors.TEST_UUID
assert uuid.UUID(str(descriptor.source_id), version=4)
descriptor = StreamDescriptor(dummy_stream, source_id=TestDescriptors.TEST_UUID_STR)
assert descriptor.source_id == TestDescriptors.TEST_UUID
descriptor = StreamDescriptor(dummy_stream, source_id=TestDescriptors.TEST_UUID)
assert descriptor.source_id == TestDescriptors.TEST_UUID
with pytest.raises(ValueError):
StreamDescriptor(dummy_stream, source_id=TestDescriptors.INVALID_UUID)
def test_uuid_file_descriptor(self):
dummy_file = "dummy"
descriptor = FileDescriptor(dummy_file)
assert descriptor.source_id
assert descriptor.source_id != TestDescriptors.TEST_UUID
assert uuid.UUID(str(descriptor.source_id), version=4)
descriptor = FileDescriptor(dummy_file, source_id=TestDescriptors.TEST_UUID_STR)
assert descriptor.source_id == TestDescriptors.TEST_UUID
descriptor = FileDescriptor(dummy_file, source_id=TestDescriptors.TEST_UUID)
assert descriptor.source_id == TestDescriptors.TEST_UUID
with pytest.raises(ValueError):
FileDescriptor(dummy_file, source_id=TestDescriptors.INVALID_UUID)
def test_uuid_blob_descriptor(self):
dummy_file = "dummy"
descriptor = BlobDescriptor(dummy_file)
assert descriptor.source_id
assert descriptor.source_id != TestDescriptors.TEST_UUID
assert uuid.UUID(str(descriptor.source_id), version=4)
descriptor = BlobDescriptor(dummy_file, source_id=TestDescriptors.TEST_UUID_STR)
assert descriptor.source_id == TestDescriptors.TEST_UUID
descriptor = BlobDescriptor(dummy_file, source_id=TestDescriptors.TEST_UUID)
assert descriptor.source_id == TestDescriptors.TEST_UUID
with pytest.raises(ValueError):
BlobDescriptor(dummy_file, source_id=TestDescriptors.INVALID_UUID)
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/test_ingestion_blob_info.py 0000664 0000000 0000000 00000014341 14172227630 0027675 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import json
import re
import unittest
from uuid import UUID
from azure.kusto.data.data_format import DataFormat
from azure.kusto.ingest import (
BlobDescriptor,
IngestionProperties,
ColumnMapping,
ReportLevel,
ReportMethod,
ValidationPolicy,
ValidationOptions,
ValidationImplications,
)
from azure.kusto.ingest.ingestion_blob_info import IngestionBlobInfo
TIMESTAMP_REGEX = "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}"
class IngestionBlobInfoTest(unittest.TestCase):
"""Tests serialization of ingestion blob info. This serialization will be queued to the DM."""
def test_blob_info_csv_mapping(self):
"""Tests serialization of csv ingestion blob info."""
validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort)
column_mapping = ColumnMapping("ColumnName", "cslDataType", ordinal=1)
properties = IngestionProperties(
database="database",
table="table",
data_format=DataFormat.CSV,
column_mappings=[column_mapping],
additional_tags=["tag"],
ingest_if_not_exists=["ingestIfNotExistTags"],
ingest_by_tags=["ingestByTags"],
drop_by_tags=["dropByTags"],
flush_immediately=True,
report_level=ReportLevel.DoNotReport,
report_method=ReportMethod.Queue,
validation_policy=validation_policy,
)
blob = BlobDescriptor("somepath", 10)
blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText")
self._verify_ingestion_blob_info_result(blob_info.to_json())
def test_blob_csv_mapping_reference(self):
"""Tests serialization of ingestion blob info with csv mapping reference."""
validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort)
properties = IngestionProperties(
database="database",
table="table",
data_format=DataFormat.CSV,
ingestion_mapping_reference="csvMappingReference",
additional_tags=["tag"],
ingest_if_not_exists=["ingestIfNotExistTags"],
ingest_by_tags=["ingestByTags"],
drop_by_tags=["dropByTags"],
flush_immediately=True,
report_level=ReportLevel.DoNotReport,
report_method=ReportMethod.Queue,
validation_policy=validation_policy,
)
blob = BlobDescriptor("somepath", 10)
blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText")
self._verify_ingestion_blob_info_result(blob_info.to_json())
def test_blob_info_json_mapping(self):
"""Tests serialization of json ingestion blob info."""
validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort)
properties = IngestionProperties(
database="database",
table="table",
data_format=DataFormat.JSON,
column_mappings=[ColumnMapping("ColumnName", "datatype", path="jsonpath")],
additional_tags=["tag"],
ingest_if_not_exists=["ingestIfNotExistTags"],
ingest_by_tags=["ingestByTags"],
drop_by_tags=["dropByTags"],
flush_immediately=True,
report_level=ReportLevel.DoNotReport,
report_method=ReportMethod.Queue,
validation_policy=validation_policy,
)
blob = BlobDescriptor("somepath", 10)
blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText")
self._verify_ingestion_blob_info_result(blob_info.to_json())
def test_blob_json_mapping_reference(self):
"""Tests serialization of ingestion blob info with json mapping reference."""
validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort)
properties = IngestionProperties(
database="database",
table="table",
data_format=DataFormat.JSON,
ingestion_mapping_reference="jsonMappingReference",
additional_tags=["tag"],
ingest_if_not_exists=["ingestIfNotExistTags"],
ingest_by_tags=["ingestByTags"],
drop_by_tags=["dropByTags"],
flush_immediately=True,
report_level=ReportLevel.DoNotReport,
report_method=ReportMethod.Queue,
validation_policy=validation_policy,
)
blob = BlobDescriptor("somepath", 10)
blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText")
self._verify_ingestion_blob_info_result(blob_info.to_json())
def _verify_ingestion_blob_info_result(self, ingestion_blob_info):
result = json.loads(ingestion_blob_info)
assert result is not None
assert isinstance(result, dict)
assert result["BlobPath"] == "somepath"
assert result["DatabaseName"] == "database"
assert result["TableName"] == "table"
assert isinstance(result["RawDataSize"], int)
assert isinstance(result["IgnoreSizeLimit"], bool)
assert isinstance(result["FlushImmediately"], bool)
assert isinstance(result["RetainBlobOnSuccess"], bool)
assert isinstance(result["ReportMethod"], int)
assert isinstance(result["ReportLevel"], int)
assert isinstance(UUID(result["Id"]), UUID)
assert re.match(TIMESTAMP_REGEX, result["SourceMessageCreationTime"])
assert result["AdditionalProperties"]["authorizationContext"] == "authorizationContextText"
assert result["AdditionalProperties"]["ingestIfNotExists"] == '["ingestIfNotExistTags"]'
assert result["AdditionalProperties"]["ValidationPolicy"] in (
'{"ValidationImplications":1,"ValidationOptions":1}',
'{"ValidationImplications":ValidationImplications.BestEffort,"ValidationOptions":ValidationOptions.ValidateCsvInputConstantColumns}',
)
assert result["AdditionalProperties"]["tags"] == '["tag","drop-by:dropByTags","ingest-by:ingestByTags"]'
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/test_ingestion_properties.py 0000664 0000000 0000000 00000005551 14172227630 0030143 0 ustar 00root root 0000000 0000000 import pytest
from azure.kusto.data.data_format import IngestionMappingKind, DataFormat
from azure.kusto.ingest import IngestionProperties, ColumnMapping, TransformationMethod
from azure.kusto.ingest.exceptions import KustoDuplicateMappingError, KustoMissingMappingError, KustoMappingError
def test_duplicate_reference_and_column_mappings_raises():
"""Tests invalid ingestion properties."""
with pytest.raises(KustoDuplicateMappingError):
IngestionProperties(
database="database", table="table", column_mappings=[ColumnMapping("test", "int")], ingestion_mapping_reference="ingestionMappingReference"
)
def test_mapping_kind_without_mapping_raises():
with pytest.raises(KustoMissingMappingError):
IngestionProperties(database="database", table="table", ingestion_mapping_kind=IngestionMappingKind.CSV)
def test_mapping_kind_data_format_mismatch():
with pytest.raises(KustoMappingError):
IngestionProperties(
database="database",
table="table",
ingestion_mapping_reference="ingestionMappingReference",
data_format=DataFormat.JSON,
ingestion_mapping_kind=IngestionMappingKind.CSV,
)
def test_mapping_kind_data_format_invalid_no_name():
with pytest.raises(KustoMappingError):
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("", "int")],
data_format=DataFormat.JSON,
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
def test_mapping_kind_data_format_invalid_no_path():
with pytest.raises(KustoMappingError):
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("test", "int")],
data_format=DataFormat.JSON,
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
def test_mapping_kind_data_format_with_path():
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("test", "int", "path")],
data_format=DataFormat.JSON,
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
def test_mapping_kind_data_format_with_transform():
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("test", "int", transform=TransformationMethod.SOURCE_LINE_NUMBER)],
data_format=DataFormat.JSON,
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
def test_mapping_kind_data_format_with_no_columns():
with pytest.raises(KustoMappingError):
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("test", "int")],
data_format=DataFormat.AVRO,
ingestion_mapping_kind=IngestionMappingKind.AVRO,
)
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/test_kusto_ingest_client.py 0000664 0000000 0000000 00000037734 14172227630 0027754 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import io
import json
import os
import uuid
from pathlib import Path
import pytest
import responses
from mock import patch
from azure.kusto.data.data_format import DataFormat
from azure.kusto.ingest import QueuedIngestClient, IngestionProperties, IngestionStatus
from azure.kusto.ingest.exceptions import KustoInvalidEndpointError
from azure.kusto.ingest.managed_streaming_ingest_client import ManagedStreamingIngestClient
pandas_installed = False
try:
import pandas
pandas_installed = True
except:
pass
UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
BLOB_NAME_REGEX = "database__table__" + UUID_REGEX + "__dataset.csv.gz"
BLOB_URL_REGEX = "https://storageaccount.blob.core.windows.net/tempstorage/database__table__" + UUID_REGEX + "__dataset.csv.gz[?]sas"
STORAGE_QUEUE_URL = "https://storageaccount.queue.core.windows.net/readyforaggregation-secured?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
TEMP_STORAGE_URL = "https://storageaccount.blob.core.windows.net/tempstorage?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
def request_callback(request):
body = json.loads(request.body.decode()) if type(request.body) == bytes else json.loads(request.body)
response_status = 400
response_headers = dict()
response_body = {}
if ".get ingestion resources" in body["csl"]:
response_status = 200
response_body = {
"Tables": [
{
"TableName": "Table_0",
"Columns": [{"ColumnName": "ResourceTypeName", "DataType": "String"}, {"ColumnName": "StorageRoot", "DataType": "String"}],
"Rows": [
[
"SecuredReadyForAggregationQueue",
STORAGE_QUEUE_URL,
],
[
"SecuredReadyForAggregationQueue",
STORAGE_QUEUE_URL,
],
[
"SecuredReadyForAggregationQueue",
STORAGE_QUEUE_URL,
],
[
"SecuredReadyForAggregationQueue",
STORAGE_QUEUE_URL,
],
[
"SecuredReadyForAggregationQueue",
STORAGE_QUEUE_URL,
],
[
"FailedIngestionsQueue",
"https://storageaccount.queue.core.windows.net/failedingestions?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
],
[
"SuccessfulIngestionsQueue",
"https://storageaccount.queue.core.windows.net/successfulingestions?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
],
[
"TempStorage",
TEMP_STORAGE_URL,
],
[
"TempStorage",
TEMP_STORAGE_URL,
],
[
"TempStorage",
TEMP_STORAGE_URL,
],
[
"TempStorage",
TEMP_STORAGE_URL,
],
[
"TempStorage",
TEMP_STORAGE_URL,
],
[
"IngestionsStatusTable",
"https://storageaccount.table.core.windows.net/ingestionsstatus?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
],
],
}
]
}
if ".get kusto identity token" in body["csl"]:
response_status = 200
response_body = {
"Tables": [{"TableName": "Table_0", "Columns": [{"ColumnName": "AuthorizationContext", "DataType": "String"}], "Rows": [["authorization_context"]]}]
}
return response_status, response_headers, json.dumps(response_body)
def request_error_callback(request):
body = json.loads(request.body.decode()) if type(request.body) == bytes else json.loads(request.body)
response_status = 400
response_headers = dict()
response_body = {}
if ".get ingestion resources" in body["csl"]:
response_status = 400
response_body = {
"error": {
"code": "BadRequest",
"message": "Request is invalid and cannot be executed.",
"@type": "Kusto.Common.Svc.Exceptions.AdminCommandWrongEndpointException",
"@message": "Cannot get ingestion resources from this service endpoint. The appropriate endpoint is most likely "
"'https://ingest-somecluster.kusto.windows.net/'.",
"@context": {
"timestamp": "2021-10-12T06:05:35.6602087Z",
"serviceAlias": "SomeCluster",
"machineName": "KEngine000000",
"processName": "Kusto.WinSvc.Svc",
"processId": 2648,
"threadId": 472,
"appDomainName": "Kusto.WinSvc.Svc.exe",
"clientRequestId": "KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674",
"activityId": "87eb8fc9-78b3-4580-bcc8-6c90482f9118",
"subActivityId": "bbfb038b-4467-4f96-afd4-945904fc6278",
"activityType": "DN.AdminCommand.IngestionResourcesGetCommand",
"parentActivityId": "00e678e9-4204-4143-8c94-6afd94c27430",
"activityStack": "(Activity stack: CRID=KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674 ARID=87eb8fc9-78b3-4580-bcc8-6c90482f9118 > DN.Admin.Client.ExecuteControlCommand/833dfb85-5d67-44b7-882d-eb2283e65780 > P.WCF.Service.ExecuteControlCommand..IInterNodeCommunicationAdminContract/3784e74f-1d89-4c15-adef-0a360c4c431e > DN.FE.ExecuteControlCommand/00e678e9-4204-4143-8c94-6afd94c27430 > DN.AdminCommand.IngestionResourcesGetCommand/bbfb038b-4467-4f96-afd4-945904fc6278)",
},
"@permanent": True,
}
}
if ".show version" in body["csl"]:
response_status = 200
response_body = {
"Tables": [
{
"TableName": "Table_0",
"Columns": [
{"ColumnName": "BuildVersion", "DataType": "String"},
{"ColumnName": "BuildTime", "DataType": "DateTime"},
{"ColumnName": "ServiceType", "DataType": "String"},
{"ColumnName": "ProductVersion", "DataType": "String"},
],
"Rows": [["1.0.0.0", "2000-01-01T00:00:00Z", "Engine", "2020-09-07 12-09-22"]],
}
]
}
return response_status, response_headers, json.dumps(response_body)
def assert_queued_upload(mock_put_message_in_queue, mock_upload_blob_from_stream, expected_url: str, check_raw_data: bool = True, format: str = "csv"):
# mock_put_message_in_queue
assert mock_put_message_in_queue.call_count == 1
put_message_in_queue_mock_kwargs = mock_put_message_in_queue.call_args_list[0][1]
queued_message_json = json.loads(put_message_in_queue_mock_kwargs["content"])
# mock_upload_blob_from_stream
# not checking the query string because it can change order, just checking it's there
assert queued_message_json["BlobPath"].startswith(expected_url) is True
assert len(queued_message_json["BlobPath"]) > len(expected_url)
assert queued_message_json["DatabaseName"] == "database"
assert queued_message_json["IgnoreSizeLimit"] is False
assert queued_message_json["AdditionalProperties"]["format"] == format
assert queued_message_json["FlushImmediately"] is False
assert queued_message_json["TableName"] == "table"
if check_raw_data:
assert queued_message_json["RawDataSize"] > 0
assert queued_message_json["RetainBlobOnSuccess"] is True
if mock_upload_blob_from_stream is not None:
upload_blob_kwargs = mock_upload_blob_from_stream.call_args_list[0][1]
assert issubclass(type(upload_blob_kwargs["data"]), io.BufferedIOBase)
@pytest.fixture(params=[QueuedIngestClient, ManagedStreamingIngestClient])
def ingest_client_class(request):
if request.param == ManagedStreamingIngestClient:
return ManagedStreamingIngestClient.from_dm_kcsb
return request.param
class TestQueuedIngestClient:
MOCKED_UUID_4 = uuid.UUID("11111111-1111-1111-1111-111111111111")
MOCKED_PID = 64
MOCKED_TIME = 100
@responses.activate
@patch(
"azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0
) # Trick to always fallback to queued ingest
@patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
@patch("azure.storage.blob.BlobClient.upload_blob")
@patch("azure.storage.queue.QueueClient.send_message")
@patch("uuid.uuid4", return_value=MOCKED_UUID_4)
def test_sanity_ingest_from_file(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad, ingest_client_class):
responses.add_callback(
responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=request_callback, content_type="application/json"
)
ingest_client = ingest_client_class("https://ingest-somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
# ensure test can work when executed from within directories
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.QUEUED
assert_queued_upload(
mock_put_message_in_queue,
mock_upload_blob_from_stream,
"https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__dataset.csv.gz?",
)
@responses.activate
@patch("azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0)
def test_ingest_from_file_wrong_endpoint(self, ingest_client_class):
responses.add_callback(
responses.POST, "https://somecluster.kusto.windows.net/v1/rest/mgmt", callback=request_error_callback, content_type="application/json"
)
ingest_client = ingest_client_class("https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
with pytest.raises(KustoInvalidEndpointError) as ex:
ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert (
ex.value.args[0] == "You are using 'DataManagement' client type, but the provided endpoint is of ServiceType 'Engine'. Initialize the "
"client with the appropriate endpoint URI: 'https://ingest-somecluster.kusto.windows.net'"
), ("Expected exception was " "not raised")
@responses.activate
@pytest.mark.skipif(not pandas_installed, reason="requires pandas")
@patch("azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0)
@patch("azure.storage.blob.BlobClient.upload_blob")
@patch("azure.storage.queue.QueueClient.send_message")
@patch("uuid.uuid4", return_value=MOCKED_UUID_4)
@patch("time.time", return_value=MOCKED_TIME)
@patch("os.getpid", return_value=MOCKED_PID)
def test_simple_ingest_from_dataframe(self, mock_pid, mock_time, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, ingest_client_class):
responses.add_callback(
responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=request_callback, content_type="application/json"
)
ingest_client = ingest_client_class("https://ingest-somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
from pandas import DataFrame
fields = ["id", "name", "value"]
rows = [[1, "abc", 15.3], [2, "cde", 99.9]]
df = DataFrame(data=rows, columns=fields)
result = ingest_client.ingest_from_dataframe(df, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.QUEUED
expected_url = "https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__df_{0}_100_11111111-1111-1111-1111-111111111111.csv.gz?".format(
id(df)
)
assert_queued_upload(mock_put_message_in_queue, mock_upload_blob_from_stream, expected_url)
@responses.activate
@patch("azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0)
@patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
@patch("azure.storage.blob.BlobClient.upload_blob")
@patch("azure.storage.queue.QueueClient.send_message")
@patch("uuid.uuid4", return_value=MOCKED_UUID_4)
def test_sanity_ingest_from_stream(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad, ingest_client_class):
responses.add_callback(
responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=request_callback, content_type="application/json"
)
ingest_client = ingest_client_class("https://ingest-somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
# ensure test can work when executed from within directories
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_stream(io.StringIO(Path(file_path).read_text()), ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.QUEUED
assert_queued_upload(
mock_put_message_in_queue,
mock_upload_blob_from_stream,
"https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__stream.gz?",
check_raw_data=False,
)
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/test_kusto_streaming_ingest_client.py 0000664 0000000 0000000 00000020413 14172227630 0032007 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import io
import json
import os
import uuid
import pytest
import responses
from azure.kusto.data.data_format import DataFormat
from azure.kusto.ingest import KustoStreamingIngestClient, IngestionProperties, IngestionStatus, ManagedStreamingIngestClient
from azure.kusto.ingest.exceptions import KustoMissingMappingError
pandas_installed = False
try:
import pandas
pandas_installed = True
except:
pass
UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
BLOB_NAME_REGEX = "database__table__" + UUID_REGEX + "__dataset.csv.gz"
BLOB_URL_REGEX = "https://storageaccount.blob.core.windows.net/tempstorage/database__table__" + UUID_REGEX + "__dataset.csv.gz[?]sas"
def request_callback(request, client_type, custom_request_id=None):
request_id = request.headers["x-ms-client-request-id"]
if custom_request_id:
assert custom_request_id == request_id
elif client_type == KustoStreamingIngestClient:
[prefix, request_uuid] = request_id.split(";")
assert prefix == "KPC.executeStreamingIngest"
uuid.UUID(request_uuid)
elif client_type == ManagedStreamingIngestClient:
assert_managed_streaming_request_id(request_id)
response_status = 200
response_headers = dict()
response_body = {
"Tables": [
{
"TableName": "Table_0",
"Columns": [
{"ColumnName": "ConsumedRecordsCount", "DataType": "Int64"},
{"ColumnName": "UpdatePolicyStatus", "DataType": "String"},
{"ColumnName": "UpdatePolicyFailureCode", "DataType": "String"},
{"ColumnName": "UpdatePolicyFailureReason", "DataType": "String"},
],
"Rows": [[0, "Inactive", "Unknown", None]],
}
]
}
return response_status, response_headers, json.dumps(response_body)
def assert_managed_streaming_request_id(request_id: str, retry: int = 0):
[prefix, request_uuid, suffix] = request_id.split(";")
assert prefix == "KPC.executeManagedStreamingIngest"
uuid.UUID(request_uuid)
assert int(suffix) == retry
@pytest.fixture(params=[KustoStreamingIngestClient, ManagedStreamingIngestClient])
def ingest_client_class(request):
if request.param == ManagedStreamingIngestClient:
return ManagedStreamingIngestClient.from_engine_kcsb
return request.param
class TestKustoStreamingIngestClient:
@responses.activate
def test_streaming_ingest_from_file(self, ingest_client_class):
responses.add_callback(
responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class)
)
ingest_client = ingest_client_class("https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
# ensure test can work when executed from within directories
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv.gz"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.JSON, ingestion_mapping_reference="JsonMapping")
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.json"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.jsonz.gz"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.TSV)
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.tsv"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
@pytest.mark.skipif(not pandas_installed, reason="requires pandas")
@responses.activate
def test_streaming_ingest_from_dataframe(self, ingest_client_class):
responses.add_callback(
responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class)
)
ingest_client = ingest_client_class("https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
from pandas import DataFrame
fields = ["id", "name", "value"]
rows = [[1, "abc", 15.3], [2, "cde", 99.9]]
df = DataFrame(data=rows, columns=fields)
result = ingest_client.ingest_from_dataframe(df, ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
@responses.activate
def test_streaming_ingest_from_stream(self, ingest_client_class):
responses.add_callback(
responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class)
)
ingest_client = ingest_client_class("https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
byte_sequence = b"56,56,56"
bytes_stream = io.BytesIO(byte_sequence)
result = ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
str_sequence = u"57,57,57"
str_stream = io.StringIO(str_sequence)
result = ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
byte_sequence = b'{"Name":"Ben","Age":"56","Weight":"75"}'
bytes_stream = io.BytesIO(byte_sequence)
with pytest.raises(KustoMissingMappingError):
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.JSON)
ingestion_properties.ingestion_mapping_reference = "JsonMapping"
result = ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
str_sequence = u'{"Name":"Ben","Age":"56","Weight":"75"}'
str_stream = io.StringIO(str_sequence)
result = ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/test_managed_streaming_ingest.py 0000664 0000000 0000000 00000040461 14172227630 0030705 0 ustar 00root root 0000000 0000000 import io
import json
import os
import uuid
from tempfile import NamedTemporaryFile
import pytest
import responses
from mock import patch
from azure.kusto.data.data_format import DataFormat
from azure.kusto.data.exceptions import KustoApiError
from azure.kusto.ingest import ManagedStreamingIngestClient, IngestionProperties, IngestionStatus, BlobDescriptor
from azure.kusto.ingest._retry import ExponentialRetry
from test_kusto_ingest_client import request_callback as queued_request_callback, assert_queued_upload
from test_kusto_streaming_ingest_client import request_callback as streaming_request_callback, assert_managed_streaming_request_id
def mock_retry(self):
self.max_attempts = ManagedStreamingIngestClient.ATTEMPT_COUNT
self.sleep_base_secs = 0
self.max_jitter_secs = 0
self.current_attempt = 0
class TransientResponseHelper:
def __init__(self, times_to_fail):
self.times_to_fail = times_to_fail
self.total_calls = 0
def transient_error_callback(helper: TransientResponseHelper, request, custom_request_id=None):
if custom_request_id:
assert request.headers["x-ms-client-request-id"] == custom_request_id
else:
assert_managed_streaming_request_id(request.headers["x-ms-client-request-id"], helper.total_calls)
response_headers = dict()
helper.total_calls += 1
if helper.total_calls <= helper.times_to_fail:
response_status = 400
response_body = {
"error": {
"code": "General_InternalServerError",
"message": "Unknown error",
"@type": "Kusto.DataNode.Exceptions.InternalServerError",
"@message": "InternalServerError",
"@context": {
"timestamp": "2021-10-21T14:12:00.7878847Z",
"serviceAlias": "SomeCluster",
"machineName": "KEngine000000",
"processName": "Kusto.WinSvc.Svc",
"processId": 3040,
"threadId": 7524,
"appDomainName": "Kusto.WinSvc.Svc.exe",
"clientRequestId": "KNC.executeStreamingIngest;a4d80ca5-8729-404b-b745-0d5555737483",
"activityId": "bace3d9d-d949-457e-b741-1d2c6bc56658",
"subActivityId": "bace3d9d-d949-457e-b741-1d2c6bc56658",
"activityType": "PO.OWIN.CallContext",
"parentActivityId": "bace3d9d-d949-457e-b741-1d2c6bc56658",
"activityStack": "(Activity stack: CRID=KNC.executeStreamingIngest;a4d80ca5-8729-404b-b745-0d5555737493 ARID=bace3d9d-d949-457e-b741-1d2c6bc56678 > PO.OWIN.CallContext/bace3d9d-d949-457e-b741-1d2c6bc56678 > PO.OWIN.CallContext/bace3d9d-d949-457e-b741-1d2c6bc56678)",
},
"@permanent": False,
}
}
else:
response_status = 200
response_body = {
"Tables": [
{
"TableName": "Table_0",
"Columns": [
{"ColumnName": "ConsumedRecordsCount", "DataType": "Int64"},
{"ColumnName": "UpdatePolicyStatus", "DataType": "String"},
{"ColumnName": "UpdatePolicyFailureCode", "DataType": "String"},
{"ColumnName": "UpdatePolicyFailureReason", "DataType": "String"},
],
"Rows": [[0, "Inactive", "Unknown", None]],
}
]
}
return response_status, response_headers, json.dumps(response_body)
class TestManagedStreamingIngestClient:
MOCKED_UUID_4 = uuid.UUID("11111111-1111-1111-1111-111111111111")
@responses.activate
@patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
@patch("azure.storage.blob.BlobClient.upload_blob")
@patch("azure.storage.queue.QueueClient.send_message")
@patch("uuid.uuid4", return_value=MOCKED_UUID_4)
def test_fallback_big_file(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad):
responses.add_callback(
responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
)
responses.add_callback(
responses.POST,
"https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
callback=streaming_request_callback,
content_type="application/json",
)
data_format = DataFormat.ORC # Using orc to avoid compression
ingest_client = ManagedStreamingIngestClient.from_dm_kcsb("https://ingest-somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=data_format)
initial_bytes = bytearray(os.urandom(5 * 1024 * 1024))
def check_bytes(data):
assert data.read() == initial_bytes
mock_upload_blob_from_stream.side_effect = check_bytes
f = NamedTemporaryFile(dir=".", mode="wb", delete=False)
try:
f.write(initial_bytes)
f.close()
result = ingest_client.ingest_from_file(f.name, ingestion_properties=ingestion_properties)
finally:
os.unlink(f.name)
assert result.status == IngestionStatus.QUEUED
assert_queued_upload(
mock_put_message_in_queue,
mock_upload_blob_from_stream,
"https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__{}?".format(
os.path.basename(f.name)
),
format=data_format.kusto_value,
)
mock_upload_blob_from_stream.assert_called()
@responses.activate
@patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
@patch("azure.storage.blob.BlobClient.upload_blob")
@patch("azure.storage.queue.QueueClient.send_message")
@patch("uuid.uuid4", return_value=MOCKED_UUID_4)
def test_fallback_big_stream(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad):
responses.add_callback(
responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
)
responses.add_callback(
responses.POST,
"https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
callback=streaming_request_callback,
content_type="application/json",
)
data_format = DataFormat.ORC # Using orc to avoid compression
ingest_client = ManagedStreamingIngestClient.from_engine_kcsb("https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=data_format)
initial_bytes = bytearray(os.urandom(5 * 1024 * 1024))
stream = io.BytesIO(initial_bytes)
def check_bytes(data):
assert data.read() == initial_bytes
mock_upload_blob_from_stream.side_effect = check_bytes
result = ingest_client.ingest_from_stream(stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.QUEUED
assert_queued_upload(
mock_put_message_in_queue,
mock_upload_blob_from_stream,
"https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__stream?",
format=data_format.kusto_value,
check_raw_data=False,
)
mock_upload_blob_from_stream.assert_called()
@responses.activate
@patch(
"azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient._create_exponential_retry",
return_value=ExponentialRetry(ManagedStreamingIngestClient.ATTEMPT_COUNT, sleep_base_secs=0, max_jitter_secs=0),
)
@patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
@patch("azure.storage.blob.BlobClient.upload_blob")
@patch("azure.storage.queue.QueueClient.send_message")
@patch("uuid.uuid4", return_value=MOCKED_UUID_4)
def test_fallback_transient_errors_limit(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad, mock_retry):
total_attempts = 3
responses.add_callback(
responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
)
ingest_client = ManagedStreamingIngestClient.from_engine_kcsb("https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table")
helper = TransientResponseHelper(times_to_fail=total_attempts)
responses.add_callback(
responses.POST,
"https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
callback=lambda request: transient_error_callback(helper, request),
content_type="application/json",
)
# ensure test can work when executed from within directories
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.QUEUED
assert_queued_upload(
mock_put_message_in_queue,
mock_upload_blob_from_stream,
"https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__dataset.csv.gz?",
)
assert helper.total_calls == total_attempts
@responses.activate
@patch(
"azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient._create_exponential_retry",
return_value=ExponentialRetry(ManagedStreamingIngestClient.ATTEMPT_COUNT, sleep_base_secs=0, max_jitter_secs=0),
)
@patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
@patch("azure.storage.blob.BlobClient.upload_blob")
@patch("azure.storage.queue.QueueClient.send_message")
@patch("uuid.uuid4", return_value=MOCKED_UUID_4)
def test_fallback_transient_single_error(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad, mock_retry):
total_failures = 2
responses.add_callback(
responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
)
helper = TransientResponseHelper(times_to_fail=total_failures)
responses.add_callback(
responses.POST,
"https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
callback=lambda request: transient_error_callback(helper, request),
content_type="application/json",
)
ingest_client = ManagedStreamingIngestClient.from_engine_kcsb("https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table")
# ensure test can work when executed from within directories
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
assert helper.total_calls == total_failures + 1
@responses.activate
def test_permanent_error(self):
responses.add(
responses.POST,
"https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
status=400,
json={
"error": {
"code": "BadRequest",
"message": "Request is invalid and cannot be executed.",
"@type": "Kusto.Common.Svc.Exceptions.AdminCommandWrongEndpointException",
"@message": "Cannot get ingestion resources from this service endpoint. The appropriate endpoint is most likely "
"'https://ingest-somecluster.kusto.windows.net/'.",
"@context": {
"timestamp": "2021-10-12T06:05:35.6602087Z",
"serviceAlias": "SomeCluster",
"machineName": "KEngine000000",
"processName": "Kusto.WinSvc.Svc",
"processId": 2648,
"threadId": 472,
"appDomainName": "Kusto.WinSvc.Svc.exe",
"clientRequestId": "KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674",
"activityId": "87eb8fc9-78b3-4580-bcc8-6c90482f9118",
"subActivityId": "bbfb038b-4467-4f96-afd4-945904fc6278",
"activityType": "DN.AdminCommand.IngestionResourcesGetCommand",
"parentActivityId": "00e678e9-4204-4143-8c94-6afd94c27430",
"activityStack": "(Activity stack: CRID=KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674 ARID=87eb8fc9-78b3-4580-bcc8-6c90482f9118 > DN.Admin.Client.ExecuteControlCommand/833dfb85-5d67-44b7-882d-eb2283e65780 > P.WCF.Service.ExecuteControlCommand..IInterNodeCommunicationAdminContract/3784e74f-1d89-4c15-adef-0a360c4c431e > DN.FE.ExecuteControlCommand/00e678e9-4204-4143-8c94-6afd94c27430 > DN.AdminCommand.IngestionResourcesGetCommand/bbfb038b-4467-4f96-afd4-945904fc6278)",
},
"@permanent": True,
}
},
content_type="application/json",
)
ingest_client = ManagedStreamingIngestClient.from_dm_kcsb("https://ingest-somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
missing_path_parts = []
for path_part in path_parts:
if path_part not in current_dir:
missing_path_parts.append(path_part)
file_path = os.path.join(current_dir, *missing_path_parts)
with pytest.raises(KustoApiError) as ex:
ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
assert ex.value.get_api_error().permanent == True
@responses.activate
@patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
@patch("azure.storage.queue.QueueClient.send_message")
@patch("uuid.uuid4", return_value=MOCKED_UUID_4)
def test_blob_ingestion(self, mock_uuid, mock_put_message_in_queue, mock_aad):
responses.add_callback(
responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
)
ingest_client = ManagedStreamingIngestClient.from_dm_kcsb("https://ingest-somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database", table="table")
blob_path = (
"https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__tmpbvk40leg?sp=rl&st=2020-05-20T13"
"%3A38%3A37Z&se=2020-05-21T13%3A38%3A37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx "
)
result = ingest_client.ingest_from_blob(BlobDescriptor(blob_path, 1), ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.QUEUED
assert_queued_upload(
mock_put_message_in_queue,
mock_upload_blob_from_stream=None,
expected_url="https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__tmpbvk40leg?",
)
azure-kusto-python-3.0.1/azure-kusto-ingest/tests/test_status_q.py 0000664 0000000 0000000 00000025443 14172227630 0025535 0 ustar 00root root 0000000 0000000 # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
import json
import time
import unittest
from uuid import uuid4
import mock
from azure.kusto.ingest import QueuedIngestClient
from azure.kusto.ingest._resource_manager import _ResourceUri
from azure.kusto.ingest.status import KustoIngestStatusQueues, SuccessMessage, FailureMessage
from azure.storage.queue import QueueMessage, QueueClient
ENDPOINT_SUFFIX = "sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
OBJECT_TYPE = "core.windows.net"
def mock_message(success):
m = QueueMessage()
m.id = uuid4()
m.insertion_time = time.time()
m.expiration_time = None
m.dequeue_count = None
if success:
content = {
"OperationId": str(m.id),
"Database": "db1",
"Table": "table1",
"IngestionSourceId": str(m.id),
"IngestionSourcePath": "blob/path",
"RootActivityId": "1",
"SucceededOn": time.time(),
}
else:
content = {
"OperationId": str(m.id),
"Database": "db1",
"Table": "table1",
"IngestionSourceId": str(m.id),
"IngestionSourcePath": "blob/path",
"RootActivityId": "1",
"FailedOn": time.time(),
"Details": "",
"ErrorCode": "1",
"FailureStatus": "",
"OriginatesFromUpdatePolicy": "",
"ShouldRetry": False,
}
m.content = json.dumps(content)
m.pop_receipt = None
m.time_next_visible = None
return m
def fake_peek_factory(f):
def fake_peek(self, max_messages):
return f(self.queue_name, max_messages)
return fake_peek
def fake_receive_factory(f):
def fake_receive(self, messages_per_page, *args, **kwargs):
return f(self.queue_name, messages_per_page)
return fake_receive
def fake_delete_factory(f):
def fake_delete(self, n):
return f(self.queue_name, n)
return fake_delete
class StatusQTests(unittest.TestCase):
def test_init(self):
client = QueuedIngestClient("some-cluster")
qs = KustoIngestStatusQueues(client)
assert qs.success.message_cls == SuccessMessage
assert qs.failure.message_cls == FailureMessage
def test_isempty(self):
client = QueuedIngestClient("some-cluster")
fake_peek = fake_peek_factory(
lambda queue_name, num_messages=1: [mock_message(success=True) for _ in range(0, num_messages)] if "qs" in queue_name else []
)
with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues") as mocked_get_success_qs, mock.patch.object(
client._resource_manager, "get_failed_ingestions_queues"
) as mocked_get_failed_qs, mock.patch.object(QueueClient, "peek_messages", autospec=True, side_effect=fake_peek) as q_mock:
fake_failed_queue = _ResourceUri(
"mocked_storage_account1",
OBJECT_TYPE,
"queue",
"mocked_qf_name",
ENDPOINT_SUFFIX,
)
fake_success_queue = _ResourceUri(
"mocked_storage_account2",
OBJECT_TYPE,
"queue",
"mocked_qs_name",
ENDPOINT_SUFFIX,
)
mocked_get_success_qs.return_value = [fake_success_queue]
mocked_get_failed_qs.return_value = [fake_failed_queue]
qs = KustoIngestStatusQueues(client)
assert qs.success.is_empty() is False
assert qs.failure.is_empty() is True
assert q_mock.call_count == 2
assert q_mock.call_args_list[0][1]["max_messages"] == 2
assert q_mock.call_args_list[1][1]["max_messages"] == 2
def test_peek(self):
client = QueuedIngestClient("some-cluster")
fake_peek = fake_peek_factory(
lambda queue_name, num_messages=1: [
mock_message(success=True) if "qs" in queue_name else mock_message(success=False) for _ in range(0, num_messages)
]
)
with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues") as mocked_get_success_qs, mock.patch.object(
client._resource_manager, "get_failed_ingestions_queues"
) as mocked_get_failed_qs, mock.patch.object(QueueClient, "peek_messages", autospec=True, side_effect=fake_peek) as q_mock:
fake_failed_queue1 = _ResourceUri(
"mocked_storage_account_f1",
OBJECT_TYPE,
"queue",
"mocked_qf_1_name",
ENDPOINT_SUFFIX,
)
fake_failed_queue2 = _ResourceUri(
"mocked_storage_account_f2",
OBJECT_TYPE,
"queue",
"mocked_qf_2_name",
ENDPOINT_SUFFIX,
)
fake_success_queue = _ResourceUri(
"mocked_storage_account2",
OBJECT_TYPE,
"queue",
"mocked_qs_name",
ENDPOINT_SUFFIX,
)
mocked_get_success_qs.return_value = [fake_success_queue]
mocked_get_failed_qs.return_value = [fake_failed_queue1, fake_failed_queue2]
qs = KustoIngestStatusQueues(client)
peek_success_actual = qs.success.peek()
peek_failure_actual = qs.failure.peek(6)
assert len(peek_success_actual) == 1
for m in peek_failure_actual:
assert isinstance(m, FailureMessage) is True
for m in peek_success_actual:
assert isinstance(m, SuccessMessage) is True
assert len(peek_failure_actual) == 6
actual = {}
assert len(QueueClient.peek_messages.call_args_list) == 3
for call_args in q_mock.call_args_list:
actual[call_args[0][0].queue_name] = actual.get(call_args[0][0].queue_name, 0) + call_args[1]["max_messages"]
assert actual[fake_failed_queue2.object_name] == 4
assert actual[fake_failed_queue1.object_name] == 4
assert actual[fake_success_queue.object_name] == 2
def test_pop(self):
client = QueuedIngestClient("some-cluster")
fake_receive = fake_receive_factory(
lambda queue_name, num_messages=1: [
mock_message(success=True) if "qs" in queue_name else mock_message(success=False) for _ in range(0, num_messages)
]
)
with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues") as mocked_get_success_qs, mock.patch.object(
client._resource_manager, "get_failed_ingestions_queues"
) as mocked_get_failed_qs, mock.patch.object(
QueueClient,
"receive_messages",
autospec=True,
side_effect=fake_receive,
) as q_receive_mock, mock.patch.object(
QueueClient, "delete_message", return_value=None
) as q_del_mock:
fake_failed_queue1 = _ResourceUri(
"mocked_storage_account_f1",
OBJECT_TYPE,
"queue",
"mocked_qf_1_name",
ENDPOINT_SUFFIX,
)
fake_failed_queue2 = _ResourceUri(
"mocked_storage_account_f2",
OBJECT_TYPE,
"queue",
"mocked_qf_2_name",
ENDPOINT_SUFFIX,
)
fake_success_queue = _ResourceUri(
"mocked_storage_account2",
OBJECT_TYPE,
"queue",
"mocked_qs_name",
ENDPOINT_SUFFIX,
)
mocked_get_success_qs.return_value = [fake_success_queue]
mocked_get_failed_qs.return_value = [fake_failed_queue1, fake_failed_queue2]
qs = KustoIngestStatusQueues(client)
get_success_actual = qs.success.pop()
get_failure_actual = qs.failure.pop(6)
assert len(get_success_actual) == 1
assert len(get_failure_actual) == 6
for m in get_failure_actual:
assert isinstance(m, FailureMessage)
for m in get_success_actual:
assert isinstance(m, SuccessMessage)
assert q_receive_mock.call_count == 3
assert q_del_mock.call_count == len(get_success_actual) + len(get_failure_actual)
assert q_receive_mock.call_args_list[0][1]["messages_per_page"] == 2
actual = {
q_receive_mock.call_args_list[1][0][0].queue_name: q_receive_mock.call_args_list[1][1]["messages_per_page"],
q_receive_mock.call_args_list[2][0][0].queue_name: q_receive_mock.call_args_list[2][1]["messages_per_page"],
}
assert actual[fake_failed_queue2.object_name] == 4
assert actual[fake_failed_queue1.object_name] == 4
def test_pop_unbalanced_queues(self):
client = QueuedIngestClient("some-cluster")
fake_receive = fake_receive_factory(
lambda queue_name, messages_per_page=1: [mock_message(success=False) for _ in range(0, messages_per_page)] if "1" in queue_name else []
)
with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues"), mock.patch.object(
client._resource_manager, "get_failed_ingestions_queues"
) as mocked_get_failed_qs, mock.patch.object(
QueueClient,
"receive_messages",
autospec=True,
side_effect=fake_receive,
) as q_receive_mock, mock.patch.object(
QueueClient, "delete_message", return_value=None
):
fake_failed_queue1 = _ResourceUri(
"mocked_storage_account_f1",
OBJECT_TYPE,
"queue",
"mocked_qf_1_name",
ENDPOINT_SUFFIX,
)
fake_failed_queue2 = _ResourceUri(
"mocked_storage_account_f2",
OBJECT_TYPE,
"queue",
"mocked_qf_2_name",
ENDPOINT_SUFFIX,
)
mocked_get_failed_qs.return_value = [fake_failed_queue1, fake_failed_queue2]
qs = KustoIngestStatusQueues(client)
get_failure_actual = qs.failure.pop(6)
assert len(get_failure_actual) == 6
for m in get_failure_actual:
assert isinstance(m, FailureMessage)
assert q_receive_mock.call_count == 3
actual = {}
for call_args in q_receive_mock.call_args_list:
actual[call_args[0][0].queue_name] = actual.get(call_args[0][0].queue_name, 0) + call_args[1]["messages_per_page"]
assert actual[fake_failed_queue2.object_name] + actual[fake_failed_queue1.object_name] == (4 + 4 + 6)
azure-kusto-python-3.0.1/azure-kusto-ingest/tox.ini 0000664 0000000 0000000 00000000131 14172227630 0022415 0 ustar 00root root 0000000 0000000 [tox]
envlist = py27,py35,py37
[testenv]
deps=mock
pytest
pandas
commands=pytest azure-kusto-python-3.0.1/back_to_black.bat 0000664 0000000 0000000 00000000071 14172227630 0020553 0 ustar 00root root 0000000 0000000 pip install --upgrade black
black . --line-length 160
azure-kusto-python-3.0.1/build_packages.py 0000664 0000000 0000000 00000004532 14172227630 0020642 0 ustar 00root root 0000000 0000000 import argparse
import os
from subprocess import check_call
from pathlib import Path
try:
from packaging.version import parse as Version, InvalidVersion
except ImportError: # Should not happen, but at worst in most case this is the same
from pip._vendor.packaging.version import parse as Version, InvalidVersion
DEFAULT_DESTINATION_FOLDER = os.path.join("..", "dist")
package_list = ["azure-kusto-data", "azure-kusto-ingest"]
def create_package(name, dest_folder=DEFAULT_DESTINATION_FOLDER):
absdirpath = os.path.abspath(name)
check_call(["python", "setup.py", "bdist_wheel", "-d", dest_folder], cwd=absdirpath)
check_call(["python", "setup.py", "sdist", "-d", dest_folder], cwd=absdirpath)
def travis_build_package():
"""Assumed called on Travis, to prepare a package to be deployed
This method prints on stdout for Travis.
Return is obj to pass to sys.exit() directly
"""
travis_tag = os.environ.get("TRAVIS_TAG")
if not travis_tag:
print("TRAVIS_TAG environment variable is not present")
return "TRAVIS_TAG environment variable is not present"
try:
version = Version(travis_tag)
except InvalidVersion:
failure = "Version must be a valid PEP440 version (version is: {})".format(version)
print(failure)
return failure
abs_dist_path = Path(os.environ["TRAVIS_BUILD_DIR"], "dist")
[create_package(package, str(abs_dist_path)) for package in package_list]
print("Produced:\n{}".format(list(abs_dist_path.glob("*"))))
pattern = "*{}*".format(version)
packages = list(abs_dist_path.glob(pattern))
if not packages:
return "Package version does not match tag {}, abort".format(version)
pypi_server = os.environ.get("PYPI_SERVER", "default PyPI server")
print("Package created as expected and will be pushed to {}".format(pypi_server))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Build Azure package.")
parser.add_argument("name", help="The package name")
parser.add_argument("--dest", "-d", default=DEFAULT_DESTINATION_FOLDER, help="Destination folder. Relative to the package dir. [default: %(default)s]")
args = parser.parse_args()
if args.name == "all":
for package in package_list:
create_package(package, args.dest)
else:
create_package(args.name, args.dest)
azure-kusto-python-3.0.1/dev_requirements.txt 0000664 0000000 0000000 00000000244 14172227630 0021451 0 ustar 00root root 0000000 0000000 pytest>=3.2.0
mock>=2.0.0
responses>=0.9.0
pandas>=0.24.0
black;python_version >= '3.6'
aioresponses>=0.6.2
pytest-asyncio>=0.12.0
azure-core>=1.11.0
asgiref>=3.2.3 azure-kusto-python-3.0.1/pyproject.toml 0000664 0000000 0000000 00000000036 14172227630 0020242 0 ustar 00root root 0000000 0000000 [tool.black]
line-length = 160 azure-kusto-python-3.0.1/quick_start/ 0000775 0000000 0000000 00000000000 14172227630 0017660 5 ustar 00root root 0000000 0000000 azure-kusto-python-3.0.1/quick_start/README.md 0000664 0000000 0000000 00000004533 14172227630 0021144 0 ustar 00root root 0000000 0000000 # Quickstart App
The quick start application is a **self-contained and runnable** example script that demonstrates authenticating, connecting to, administering, ingesting data into and querying Azure Data Explorer using the azure-kusto-python SDK.
You can use it as a baseline to write your own first kusto client application, altering the code as you go, or copy code sections out of it into your app.
**Tip:** The app includes comments with tips on recommendations, coding best practices, links to reference materials and recommended TODO changes when adapting the code to your needs.
## Using the App for the first time
### Prerequisites
1. Set up Python version 3.7 or higher on your machine. For instructions, consult a Python environment setup tutorial, like [this](https://www.
tutorialspoint.com/java/java_environment_setup.htm).
### Retrieving the app from GitHub
1. Download the app files from this GitHub repo.
1. Modify the `kusto_sample_config.json` file, changing `KustoUri`, `IngestUri` and `DatabaseName` appropriately for your cluster.
### Retrieving the app from OneClick
1. Open a browser and type your cluster's URL (e.g. https://mycluster.westeurope.kusto.windows.net/). You will be redirected to the _Azure Data Explorer_ Web UI.
1. On the left menu, select **Data**.
1. Click **Generate Sample App Code** and then follow the instructions in the wizard.
1. Download the app as a ZIP file.
1. Extract the app source code.
**Note**: The configuration parameters defined in the `kusto_sample_config.json` file are preconfigured with the appropriate values for your cluster. Verify that these are correct.
### Run the app
1. Open a command line window and navigate to the folder where you extracted the script.
1. Run `python -m pip install azure-kusto-data azure-kusto-ingest`. If upgrading, append ` --upgrade`.
1. Run `python kusto_sample_app.py`.
#### Troubleshooting
* If you are having trouble running the script from your IDE, first check if the script runs from the command line, then consult the troubleshooting references of your IDE.
### Optional changes
You can make the following changes from within the script itself:
- Change the default User-Prompt authentication method by editing `AUTHENTICATION_MODE`.
- Change the script to run without stopping between steps by setting `WAIT_FOR_USER = False` azure-kusto-python-3.0.1/quick_start/dataset.csv 0000664 0000000 0000000 00000002753 14172227630 0022031 0 ustar 00root root 0000000 0000000 0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,Zero,0,00:00:00,,null
1,00000001-0000-0000-0001-020304050607,1.0001,1.01,1,1,1,1,1,1,1,1,2015-01-01T01:01:01.0000000Z,One,One,1,00:01.0,,"{""rowId"": 1, ""arr"": [0,1]}"
2,00000002-0000-0000-0001-020304050607,2.0002,2.02,0,2,2,2,2,2,2,2,2016-01-01T01:01:01.0000000Z,Two,Two,2,-00:00:02.0020002,,"{""rowId"": 2, ""arr"": [0,2]}"
3,00000003-0000-0000-0001-020304050607,3.0003,3.03,1,3,3,3,3,3,3,3,2017-01-01T01:01:01.0000000Z,Three,Three,3,00:03.0,,"{""rowId"": 3, ""arr"": [0,3]}"
4,00000004-0000-0000-0001-020304050607,4.0004,4.04,0,4,4,4,4,4,4,4,2018-01-01T01:01:01.0000000Z,Four,Four,4,-00:00:04.0040004,,"{""rowId"": 4, ""arr"": [0,4]}"
5,00000005-0000-0000-0001-020304050607,5.0005,5.05,1,5,5,5,5,5,5,5,2019-01-01T01:01:01.0000000Z,Five,Five,5,00:05.0,,"{""rowId"": 5, ""arr"": [0,5]}"
6,00000006-0000-0000-0001-020304050607,6.0006,6.06,0,6,6,6,6,6,6,6,2020-01-01T01:01:01.0000000Z,Six,Six,6,-00:00:06.0060006,,"{""rowId"": 6, ""arr"": [0,6]}"
7,00000007-0000-0000-0001-020304050607,7.0007,7.07,1,7,7,7,7,7,7,7,2021-01-01T01:01:01.0000000Z,Seven,Seven,7,00:07.0,,"{""rowId"": 7, ""arr"": [0,7]}"
8,00000008-0000-0000-0001-020304050607,8.0008,8.08,0,8,8,8,8,8,8,8,2022-01-01T01:01:01.0000000Z,Eight,Eight,8,-00:00:08.0080008,,"{""rowId"": 8, ""arr"": [0,8]}"
9,00000009-0000-0000-0001-020304050607,9.0009,9.09,1,9,9,9,9,9,9,9,2023-01-01T01:01:01.0000000Z,Nine,Nine,9,00:09.0,,"{""rowId"": 9, ""arr"": [0,9]}"
azure-kusto-python-3.0.1/quick_start/dataset.json 0000664 0000000 0000000 00000001410 14172227630 0022174 0 ustar 00root root 0000000 0000000 {"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}
{"rownumber": 1, "rowguid": "00000001-0000-0000-0001-020304050607", "xdouble": 1.00001, "xfloat": 1.01, "xbool": 1, "xint16": 1, "xint32": 1, "xint64": 1, "xuint8": 1, "xuint16": 1, "xuint32": 1, "xuint64": 1, "xdate": "2015-01-01T01:01:01Z", "xsmalltext": "One", "xtext": "One", "xnumberAsText": "1", "xtime": "00:00:01.0010001", "xtextWithNulls": null, "xdynamicWithNulls": "{\"rowId\":1,\"arr\":[0,1]}"} azure-kusto-python-3.0.1/quick_start/kusto_sample_app.py 0000664 0000000 0000000 00000057746 14172227630 0023623 0 ustar 00root root 0000000 0000000 import json
import os
import time
import uuid
from distutils.util import strtobool
from typing import ClassVar
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties
from azure.kusto.data.data_format import DataFormat
from azure.kusto.data.exceptions import KustoClientError, KustoServiceError
from azure.kusto.ingest import (
QueuedIngestClient,
IngestionProperties,
FileDescriptor,
BlobDescriptor,
BaseIngestClient,
)
class KustoSampleApp:
# TODO (config - optional): Change the authentication method from "User Prompt" to any of the other options
# Some of the auth modes require additional environment variables to be set in order to work (see usage in generate_connection_string below)
# Managed Identity Authentication only works when running as an Azure service (webapp, function, etc.)
AUTHENTICATION_MODE = "UserPrompt" # Options: (UserPrompt|ManagedIdentity|AppKey|AppCertificate)
# TODO (config - optional): Toggle to False to execute this script "unattended"
WAIT_FOR_USER = True
# TODO (config):
# If this quickstart app was downloaded from OneClick, kusto_sample_config.json should be pre-populated with your cluster's details
# If this quickstart app was downloaded from GitHub, edit kusto_sample_config.json and modify the cluster URL and database fields appropriately
CONFIG_FILE_NAME = "kusto_sample_config.json"
BATCHING_POLICY = '{ "MaximumBatchingTimeSpan": "00:00:10", "MaximumNumberOfItems": 500, "MaximumRawDataSizeMB": 1024 }'
WAIT_FOR_INGEST_SECONDS = 20
_step = 1
use_existing_table: ClassVar[bool]
database_name: ClassVar[str]
table_name: ClassVar[str]
table_schema: ClassVar[str]
kusto_url: ClassVar[str]
ingest_url: ClassVar[str]
data_to_ingest: ClassVar[list]
should_alter_table: ClassVar[bool]
should_query_data: ClassVar[bool]
should_ingest_data: ClassVar[bool]
@classmethod
def start(cls) -> None:
cls.load_configs(cls.CONFIG_FILE_NAME)
if cls.AUTHENTICATION_MODE == "UserPrompt":
cls.wait_for_user_to_proceed("You will be prompted *twice* for credentials during this script. Please return to the console after authenticating.")
kusto_connection_string = cls.generate_connection_string(cls.kusto_url, cls.AUTHENTICATION_MODE)
ingest_connection_string = cls.generate_connection_string(cls.ingest_url, cls.AUTHENTICATION_MODE)
# Tip: Avoid creating a new Kusto/ingest client for each use. Instead, create the clients once and reuse them.
kusto_client = KustoClient(kusto_connection_string)
ingest_client = QueuedIngestClient(ingest_connection_string)
if cls.use_existing_table:
if cls.should_alter_table:
# Tip: Usually table was originally created with a schema appropriate for the data being ingested, so this wouldn't be needed.
# Learn More: For more information about altering table schemas, see: https://docs.microsoft.com/azure/data-explorer/kusto/management/alter-table-command
cls.alter_merge_existing_table_to_provided_schema(kusto_client, cls.database_name, cls.table_name, cls.table_schema)
if cls.should_query_data:
# Learn More: For more information about Kusto Query Language (KQL), see: https://docs.microsoft.com/azure/data-explorer/write-queries
cls.query_existing_number_of_rows(kusto_client, cls.database_name, cls.table_name)
else:
# Tip: This is generally a one-time configuration
# Learn More: For more information about creating tables, see: https://docs.microsoft.com/azure/data-explorer/one-click-table
cls.create_new_table(kusto_client, cls.database_name, cls.table_name, cls.table_schema)
if cls.should_ingest_data:
for file in cls.data_to_ingest:
data_format = DataFormat[str(file["format"]).upper()]
mapping_name = file["mappingName"]
# Tip: This is generally a one-time configuration.
# Learn More: For more information about providing inline mappings and mapping references, see: https://docs.microsoft.com/azure/data-explorer/kusto/management/mappings
if not cls.create_ingestion_mappings(
strtobool(str(file["useExistingMapping"]).lower()),
kusto_client,
cls.database_name,
cls.table_name,
mapping_name,
file["mappingValue"],
data_format,
):
continue
# Learn More: For more information about ingesting data to Kusto in Python, see: https://docs.microsoft.com/azure/data-explorer/python-ingest-data
cls.ingest(file, data_format, ingest_client, cls.database_name, cls.table_name, mapping_name)
cls.wait_for_ingestion_to_complete()
if cls.should_query_data:
cls.execute_validation_queries(kusto_client, cls.database_name, cls.table_name, cls.should_ingest_data)
@classmethod
def wait_for_user_to_proceed(cls, upcoming_operation: str) -> None:
print()
print(f"Step {cls._step}: {upcoming_operation}")
cls._step = cls._step + 1
if cls.WAIT_FOR_USER:
input("Press ENTER to proceed with this operation...")
@classmethod
def die(cls, error: str, ex: Exception = None) -> None:
print(f"Script failed with error: {error}")
if ex is not None:
print("Exception:")
print(ex)
exit(-1)
@classmethod
def load_configs(cls, config_file_name: str) -> None:
try:
with open(config_file_name, "r") as config_file:
config = json.load(config_file)
except Exception as ex:
cls.die(f"Couldn't read load config file from file '{config_file_name}'", ex)
cls.use_existing_table = strtobool(str(config["useExistingTable"]).lower())
cls.database_name = config["databaseName"]
cls.table_name = config["tableName"]
cls.table_schema = config["tableSchema"]
cls.kusto_url = config["kustoUri"]
cls.ingest_url = config["ingestUri"]
cls.data_to_ingest = config["data"]
cls.should_alter_table = strtobool(str(config["alterTable"]).lower())
cls.should_query_data = strtobool(str(config["queryData"]).lower())
cls.should_ingest_data = strtobool(str(config["ingestData"]).lower())
if (
cls.database_name is None
or cls.table_name is None
or cls.table_schema is None
or cls.kusto_url is None
or cls.ingest_url is None
or cls.data_to_ingest is None
):
cls.die(f"File '{config_file_name}' is missing required fields")
@classmethod
def generate_connection_string(cls, cluster_url: str, authentication_mode: str) -> KustoConnectionStringBuilder:
# Learn More: For additional information on how to authorize users and apps in Kusto, see: https://docs.microsoft.com/azure/data-explorer/manage-database-permissions
if authentication_mode == "UserPrompt":
# Prompt user for credentials
return KustoConnectionStringBuilder.with_interactive_login(cluster_url)
elif authentication_mode == "ManagedIdentity":
return cls.create_managed_identity_connection_string(cluster_url)
elif authentication_mode == "AppKey":
# Learn More: For information about how to procure an AAD Application, see: https://docs.microsoft.com/azure/data-explorer/provision-azure-ad-app
# TODO (config - optional): App ID & tenant, and App Key to authenticate with
return KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster_url, os.environ.get("APP_ID"), os.environ.get("APP_KEY"), os.environ.get("APP_TENANT")
)
elif authentication_mode == "AppCertificate":
return cls.create_application_certificate_connection_string(cluster_url)
else:
cls.die(f"Authentication mode '{authentication_mode}' is not supported")
@classmethod
def create_managed_identity_connection_string(cls, cluster_url: str) -> KustoConnectionStringBuilder:
# Connect using the system- or user-assigned managed identity (Azure service only)
# TODO (config - optional): Managed identity client ID if you are using a user-assigned managed identity
client_id = os.environ.get("MANAGED_IDENTITY_CLIENT_ID")
if client_id is None:
return KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster_url)
else:
return KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster_url, client_id=client_id)
@classmethod
def create_application_certificate_connection_string(cls, cluster_url: str) -> KustoConnectionStringBuilder:
# TODO (config - optional): App ID & tenant, path to public certificate and path to private certificate pem file to authenticate with
app_id = os.environ.get("APP_ID")
app_tenant = os.environ.get("APP_TENANT")
private_key_pem_file_path = os.environ.get("PRIVATE_KEY_PEM_FILE_PATH")
cert_thumbprint = os.environ.get("CERT_THUMBPRINT")
public_cert_file_path = os.environ.get("PUBLIC_CERT_FILE_PATH") # Only used for "Subject Name and Issuer" auth
public_certificate = None
pem_certificate = None
try:
with open(private_key_pem_file_path, "r") as pem_file:
pem_certificate = pem_file.read()
except Exception as ex:
cls.die(f"Failed to load PEM file from {private_key_pem_file_path}", ex)
if public_cert_file_path:
try:
with open(public_cert_file_path, "r") as cert_file:
public_certificate = cert_file.read()
except Exception as ex:
cls.die(f"Failed to load public certificate file from {public_cert_file_path}", ex)
return KustoConnectionStringBuilder.with_aad_application_certificate_sni_authentication(
cluster_url, app_id, pem_certificate, public_certificate, cert_thumbprint, app_tenant
)
else:
return KustoConnectionStringBuilder.with_aad_application_certificate_authentication(
cluster_url, app_id, pem_certificate, cert_thumbprint, app_tenant
)
@classmethod
def create_new_table(cls, kusto_client: KustoClient, database_name: str, table_name: str, table_schema: str) -> None:
cls.wait_for_user_to_proceed(f"Create table '{database_name}.{table_name}'")
command = f".create table {table_name} {table_schema}"
if not cls.execute_control_command(kusto_client, database_name, command):
cls.die(f"Failed to create table or validate it exists using command '{command}'")
# Learn More:
# Kusto batches data for ingestion efficiency. The default batching policy ingests data when one of the following conditions are met:
# 1) More than 1,000 files were queued for ingestion for the same table by the same user
# 2) More than 1GB of data was queued for ingestion for the same table by the same user
# 3) More than 5 minutes have passed since the first file was queued for ingestion for the same table by the same user
# For more information about customizing the ingestion batching policy, see: https://docs.microsoft.com/azure/data-explorer/kusto/management/batchingpolicy
# Disabled to prevent an existing batching policy from being unintentionally changed
if False:
cls.alter_batching_policy(kusto_client, database_name, table_name)
@classmethod
def alter_merge_existing_table_to_provided_schema(cls, kusto_client: KustoClient, database_name: str, table_name: str, table_schema: str) -> None:
cls.wait_for_user_to_proceed(f"Alter-merge existing table '{database_name}.{table_name}' to align with the provided schema")
command = f".alter-merge table {table_name} {table_schema}"
if not cls.execute_control_command(kusto_client, database_name, command):
cls.die(f"Failed to alter table using command '{command}'")
@classmethod
def execute_control_command(cls, client: KustoClient, database_name: str, control_command: str) -> bool:
try:
client_request_properties = cls.create_client_request_properties("Python_SampleApp_ControlCommand")
result = client.execute_mgmt(database_name, control_command, client_request_properties)
# Tip: Actual implementations wouldn't generally print the response from a control command. We print here to demonstrate what the response looks like.
print(f"Response from executed control command '{control_command}':")
for row in result.primary_results[0]:
print(row.to_list())
return True
except KustoClientError as ex:
print(f"Client error while trying to execute control command '{control_command}' on database '{database_name}'")
print(ex)
except KustoServiceError as ex:
print(f"Server error while trying to execute control command '{control_command}' on database '{database_name}'")
print(ex)
except Exception as ex:
print(f"Unknown error while trying to execute control command '{control_command}' on database '{database_name}'")
print(ex)
return False
@classmethod
def execute_query(cls, kusto_client: KustoClient, database_name: str, query: str):
try:
client_request_properties = cls.create_client_request_properties("Python_SampleApp_Query")
result = kusto_client.execute_query(database_name, query, client_request_properties)
print(f"Response from executed query '{query}':")
for row in result.primary_results[0]:
print(row.to_list())
return True
except KustoClientError as ex:
print(f"Client error while trying to execute query '{query}' on database '{database_name}'")
print(ex)
except KustoServiceError as ex:
print(f"Server error while trying to execute query '{query}' on database '{database_name}'")
print(ex)
except Exception as ex:
print(f"Unknown error while trying to execute query '{query}' on database '{database_name}'")
print(ex)
return False
@classmethod
def create_client_request_properties(cls, scope: str, timeout: str = None) -> ClientRequestProperties:
client_request_properties = ClientRequestProperties()
client_request_properties.client_request_id = f"{scope};{str(uuid.uuid4())}"
client_request_properties.application = "kusto_sample_app.py"
# Tip: Though uncommon, you can alter the request default command timeout using the below command, e.g. to set the timeout to 10 minutes, use "10m"
if timeout is not None:
client_request_properties.set_option(ClientRequestProperties.request_timeout_option_name, timeout)
return client_request_properties
@classmethod
def query_existing_number_of_rows(cls, kusto_client: KustoClient, database_name: str, table_name: str) -> None:
cls.wait_for_user_to_proceed(f"Get existing row count in '{database_name}.{table_name}'")
cls.execute_query(kusto_client, database_name, f"{table_name} | count")
@classmethod
def alter_batching_policy(cls, kusto_client: KustoClient, database_name: str, table_name: str) -> None:
# Tip 1: Though most users should be fine with the defaults, to speed up ingestion, such as during development and
# in this sample app, we opt to modify the default ingestion policy to ingest data after at most 10 seconds.
# Tip 2: This is generally a one-time configuration.
# Tip 3: You can also skip the batching for some files using the Flush-Immediately property, though this option should be used with care as it is inefficient.
cls.wait_for_user_to_proceed(f"Alter the batching policy for table '{database_name}.{table_name}'")
command = f".alter table {table_name} policy ingestionbatching @'{cls.BATCHING_POLICY}'"
if not cls.execute_control_command(kusto_client, database_name, command):
print(
"Failed to alter the ingestion policy, which could be the result of insufficient permissions. The sample will still run, though ingestion will be delayed for up to 5 minutes."
)
@classmethod
def create_ingestion_mappings(
cls,
use_existing_mapping: bool,
kusto_client: KustoClient,
database_name: str,
table_name: str,
mapping_name: str,
mapping_value: str,
data_format: DataFormat,
) -> bool:
if not use_existing_mapping:
if data_format._mapping_required and not mapping_value:
print(
f"The data format '{data_format.kusto_value}' requires a mapping, but configuration indicates to not use an existing mapping and no mapping was provided. Skipping this ingestion."
)
return False
if mapping_value:
ingestion_mapping_kind = data_format.ingestion_mapping_kind.value.lower()
cls.wait_for_user_to_proceed(f"Create a '{ingestion_mapping_kind}' mapping reference named '{mapping_name}'")
if not mapping_name:
mapping_name = "DefaultQuickstartMapping" + str(uuid.UUID())[:5]
mapping_command = f".create-or-alter table {table_name} ingestion {ingestion_mapping_kind} mapping '{mapping_name}' '{mapping_value}'"
if not cls.execute_control_command(kusto_client, database_name, mapping_command):
print(f"Failed to create a '{ingestion_mapping_kind}' mapping reference named '{mapping_name}'. Skipping this ingestion.")
return False
elif data_format._mapping_required and not mapping_name:
print(
f"The data format '{data_format.kusto_value}' requires a mapping and the configuration indicates an existing mapping should be used, but none was provided. Skipping this ingestion."
)
return False
return True
@classmethod
def ingest(cls, file: dict, data_format: DataFormat, ingest_client: BaseIngestClient, database_name: str, table_name: str, mapping_name: str) -> None:
source_type = str(file["sourceType"]).lower()
uri = file["dataSourceUri"]
cls.wait_for_user_to_proceed(f"Ingest '{uri}' from '{source_type}'")
# Tip: When ingesting json files, if each line represents a single-line json, use MULTIJSON format even if the file only contains one line.
# If the json contains whitespace formatting, use SINGLEJSON. In this case, only one data row json object is allowed per file.
data_format = DataFormat.MULTIJSON if data_format == data_format.JSON else data_format
# Tip: Kusto's Python SDK can ingest data from files, blobs, open streams and pandas dataframes.
# See the SDK's samples and the E2E tests in azure.kusto.ingest for additional references.
if source_type == "localfilesource":
cls.ingest_from_file(ingest_client, database_name, table_name, uri, data_format, mapping_name)
elif source_type == "blobsource":
ingest_client: QueuedIngestClient
cls.ingest_from_blob(ingest_client, database_name, table_name, uri, data_format, mapping_name)
else:
print(f"Unknown source '{source_type}' for file '{uri}'")
@classmethod
def ingest_from_file(
cls, ingest_client: BaseIngestClient, database_name: str, table_name: str, file_path: str, data_format: DataFormat, mapping_name: str = None
):
ingestion_properties = cls.create_ingestion_properties(database_name, table_name, data_format, mapping_name)
# Tip 1: For optimal ingestion batching and performance, specify the uncompressed data size in the file descriptor instead of the default below of 0.
# Otherwise, the service will determine the file size, requiring an additional s2s call, and may not be accurate for compressed files.
# Tip 2: To correlate between ingestion operations in your applications and Kusto, set the source ID and log it somewhere
file_descriptor = FileDescriptor(file_path, size=0, source_id=uuid.uuid4())
ingest_client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_properties)
@classmethod
def ingest_from_blob(
cls, client: QueuedIngestClient, database_name: str, table_name: str, blob_url: str, data_format: DataFormat, mapping_name: str = None
):
ingestion_properties = cls.create_ingestion_properties(database_name, table_name, data_format, mapping_name)
# Tip 1: For optimal ingestion batching and performance, specify the uncompressed data size in the file descriptor instead of the default below of 0.
# Otherwise, the service will determine the file size, requiring an additional s2s call, and may not be accurate for compressed files.
# Tip 2: To correlate between ingestion operations in your applications and Kusto, set the source ID and log it somewhere
blob_descriptor = BlobDescriptor(blob_url, size=0, source_id=str(uuid.uuid4()))
client.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_properties)
@classmethod
def create_ingestion_properties(cls, database_name: str, table_name: str, data_format: DataFormat, mapping_name: str) -> IngestionProperties:
return IngestionProperties(
database=f"{database_name}",
table=f"{table_name}",
ingestion_mapping_reference=mapping_name,
# Learn More: For more information about supported data formats, see: https://docs.microsoft.com/azure/data-explorer/ingestion-supported-formats
data_format=data_format,
# TODO (config - optional): Setting the ingestion batching policy takes up to 5 minutes to take effect.
# We therefore set Flush-Immediately for the sake of the sample, but it generally shouldn't be used in practice.
# Comment out the line below after running the sample the first few times.
flush_immediately=True,
)
@classmethod
def wait_for_ingestion_to_complete(cls) -> None:
print(
f"Sleeping {cls.WAIT_FOR_INGEST_SECONDS} seconds for queued ingestion to complete. Note: This may take longer depending on the file size and ingestion batching policy."
)
for x in range(cls.WAIT_FOR_INGEST_SECONDS, 0, -1):
print(f"{x} ", end="\r")
time.sleep(1)
print()
print()
@classmethod
def execute_validation_queries(cls, kusto_client: KustoClient, database_name: str, table_name: str, should_ingest_data: bool) -> None:
optional_post_ingestion_message = "post-ingestion " if should_ingest_data else ""
print(f"Step {cls._step}: Get {optional_post_ingestion_message}row count for '{database_name}.{table_name}':")
cls._step = cls._step + 1
cls.execute_query(kusto_client, database_name, f"{table_name} | count")
print("")
print(f"Step {cls._step}: Get sample (2 records) of {optional_post_ingestion_message}data:")
cls._step = cls._step + 1
cls.execute_query(kusto_client, database_name, f"{table_name} | take 2")
def main():
print("Kusto sample app is starting...")
KustoSampleApp.start()
print("Kusto sample app done")
if __name__ == "__main__":
main()
azure-kusto-python-3.0.1/quick_start/kusto_sample_config.json 0000664 0000000 0000000 00000005256 14172227630 0024616 0 ustar 00root root 0000000 0000000 {
"kustoUri" : "https://sdkse2etest.eastus.kusto.windows.net",
"ingestUri" : "https://ingest-sdkse2etest.eastus.kusto.windows.net",
"databaseName" : "e2e",
"tableName" : "SampleTable",
"useExistingTable": false,
"alterTable": true,
"queryData": true,
"ingestData": true,
"tableSchema" : "(rownumber:int, rowguid:string, xdouble:real, xfloat:real, xbool:bool, xint16:int, xint32:int, xint64:long, xuint8:long, xuint16:long, xuint32:long, xuint64:long, xdate:datetime, xsmalltext:string, xtext:string, xnumberAsText:string, xtime:timespan, xtextWithNulls:string, xdynamicWithNulls:dynamic)",
"data" :
[
{
"sourceType": "localFileSource",
"dataSourceUri": "dataset.csv",
"format": "CSV",
"useExistingMapping": true,
"mappingName": "",
"mappingValue": ""
},
{
"sourceType": "localFileSource",
"dataSourceUri": "dataset.json",
"format": "MULTIJSON",
"useExistingMapping": false,
"mappingName": "SampleTableMapping",
"mappingValue": "[{\"Properties\":{\"Path\":\"$.rownumber\"},\"column\":\"rownumber\",\"datatype\":\"int\"}, {\"Properties\":{\"Path\":\"$.rowguid\"},\"column\":\"rowguid\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.xdouble\"},\"column\":\"xdouble\",\"datatype\":\"real\"}, {\"Properties\":{\"Path\":\"$.xfloat\"},\"column\":\"xfloat\",\"datatype\":\"real\"}, {\"Properties\":{\"Path\":\"$.xbool\"},\"column\":\"xbool\",\"datatype\":\"bool\"}, {\"Properties\":{\"Path\":\"$.xint16\"},\"column\":\"xint16\",\"datatype\":\"int\"}, {\"Properties\":{\"Path\":\"$.xint32\"},\"column\":\"xint32\",\"datatype\":\"int\"}, {\"Properties\":{\"Path\":\"$.xint64\"},\"column\":\"xint64\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xuint8\"},\"column\":\"xuint8\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xuint16\"},\"column\":\"xuint16\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xuint32\"},\"column\":\"xuint32\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xuint64\"},\"column\":\"xuint64\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xdate\"},\"column\":\"xdate\",\"datatype\":\"datetime\"}, {\"Properties\":{\"Path\":\"$.xsmalltext\"},\"column\":\"xsmalltext\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.xtext\"},\"column\":\"xtext\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.rowguid\"},\"column\":\"xnumberAsText\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.xtime\"},\"column\":\"xtime\",\"datatype\":\"timespan\"}, {\"Properties\":{\"Path\":\"$.xtextWithNulls\"},\"column\":\"xtextWithNulls\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.xdynamicWithNulls\"},\"column\":\"xdynamicWithNulls\",\"datatype\":\"dynamic\"}]"
}
]
} azure-kusto-python-3.0.1/quick_start/oneclick_instruction_box.md 0000664 0000000 0000000 00000001453 14172227630 0025305 0 ustar 00root root 0000000 0000000 ### Prerequisites
1. Set up Python version 3.7 or higher on your machine. For instructions, consult a Python environment setup tutorial, like [this](https://www.
tutorialspoint.com/java/java_environment_setup.htm).
### Instructions
1. Download the **DOWNLOAD_LINK** as a ZIP file.
1. Extract the app source code.
1. Open a command line window and navigate to the folder where you extracted the script.
1. Run `python -m pip install azure-kusto-data azure-kusto-ingest`. If upgrading, append ` --upgrade`.
1. Run `python kusto_sample_app.py`. It will already be configured to your cluster and source.
### Troubleshooting
* If you are having trouble running the script from your IDE, first check if the script runs from command line, then consult the troubleshooting references of your IDE. azure-kusto-python-3.0.1/setup.cfg 0000664 0000000 0000000 00000000227 14172227630 0017151 0 ustar 00root root 0000000 0000000 [bdist_wheel]
universal=1
[flake8]
ignore = E226,E302,E41
max-line-length = 160
exclude = tests/*
max-complexity = 10
[pylint]
max-line-length = 160 azure-kusto-python-3.0.1/setup.py 0000664 0000000 0000000 00000000312 14172227630 0017035 0 ustar 00root root 0000000 0000000 import sys
if "travis_deploy" in sys.argv:
import build_packages
sys.exit(build_packages.travis_build_package())
else:
raise ValueError("Setup file is written to support travis publish.")
azure-kusto-python-3.0.1/test.bat 0000664 0000000 0000000 00000000405 14172227630 0016775 0 ustar 00root root 0000000 0000000 cd %PROJECTS_HOME%\azure-kusto-python
call workon kusto
call pip uninstall azure-kusto-data azure-kusto-ingest -y
call pip install ./azure-kusto-data[pandas] ./azure-kusto-ingest[pandas]
call pip install --force-reinstall azure-nspkg==2.0.0
call pytest
pause