diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 0000000..754475b
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,54 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+### Describe the Bug
+A clear and concise description of what the bug is.
+
+### To Reproduce
+Steps to reproduce the behavior:
+1.
+2.
+3.
+
+### Expected Behavior
+A clear and concise description of what you expected to happen.
+
+### Environment
+Please provide the following details about your environment:
+
+- **Operating System**:
+- **Rust Version**: (e.g., 1.64.0)
+- **Project Version/Commit**: (e.g., 0.1.0 or specific commit hash)
+- **Dependencies**: (e.g., versions of dependencies, if relevant)
+
+### Configuration (if applicable)
+Provide any relevant configuration details, such as:
+
+### Configuration File (e.g., `config.toml`)
+```toml
+# Your configuration here
+```
+
+### Docker Setup (if applicable)
+Please include any Docker or container-related configuration that is relevant to the bug.
+
+### Topology (if applicable)
+```yaml
+# Your topology.yaml or equivalent file
+```
+
+### Logs and Debug Information
+If possible, provide logs or debug information that might help in diagnosing the issue.
+
+```text
+# Your log output here
+```
+
+### Additional Context
+Add any other context about the problem here, including potential workarounds or related issues.
\ No newline at end of file
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 0000000..16b479f
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+### Is your feature request related to a problem? Please describe.
+A clear and concise description of what the problem is.
+
+### Describe the solution you'd like
+A clear and concise description of what you want to happen.
+
+### Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+### Additional Context
+Add any other context or screenshots about the feature request here.
\ No newline at end of file
diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml
index 6355d40..5314356 100644
--- a/.github/workflows/build_and_test.yaml
+++ b/.github/workflows/build_and_test.yaml
@@ -35,9 +35,9 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
# rust-cache already handles all the sane defaults for caching rust builds.
- # However because we are running seperate debug/release builds in parallel,
- # we also need to add the runner and cargo_flags to the key so that a seperate cache is used.
- # Otherwise only the last build to finish would get saved to the cache.
+ # However, because we are running separate debug/release builds in parallel,
+ # we also need to add the runner and cargo_flags to the key so that a separate cache is used.
+ # Otherwise, only the last build to finish would get saved to the cache.
key: ${{ matrix.runner }} - ${{ matrix.cargo_flags }}
- name: Install external deps for the prom-remote-api crate
run: sudo apt-get install protobuf-compiler
@@ -48,10 +48,16 @@ jobs:
- name: Ensure that all crates compile and have no warnings under every possible combination of features
# some things to explicitly point out:
# * clippy also reports rustc warnings and errors
- # * clippy --all-targets causes clippy to run against tests and examples which it doesnt do by default.
- run: cargo hack --feature-powerset clippy --all-targets --locked ${{ matrix.cargo_flags }} -- -D warnings
+ # * clippy --all-targets causes clippy to run against tests and examples which it doesn't do by default.
+ run: |
+ # Display all clippy lint failures as warnings
+ cargo hack --feature-powerset clippy --all-targets --locked ${{ matrix.cargo_flags }}
+ # Fail CI on the first crate to fail clippy lints
+ cargo hack --feature-powerset clippy --all-targets --locked ${{ matrix.cargo_flags }} -- -D warnings
- name: Ensure that tests pass
- run: cargo test ${{ matrix.cargo_flags }} --all-features --all-targets -- --nocapture
+ run: |
+ cargo test --doc ${{ matrix.cargo_flags }} --all-features -- --show-output --nocapture
+ cargo test ${{ matrix.cargo_flags }} --all-features --all-targets -- --nocapture
- name: Ensure that tests did not create or modify any files that arent .gitignore'd
run: |
if [ -n "$(git status --porcelain)" ]; then
diff --git a/.gitignore b/.gitignore
index d54cb23..0ac2ed4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
/target
*.flac
.vscode/settings.json
+
+.idea
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000..67fe8ce
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+ and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+ community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+ any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+ without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[INSERT CONTACT METHOD].
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..6ba943a
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,105 @@
+# Contributing to BRRO Compressor
+
+Thank you for considering contributing to the BRRO Compressor project! Whether it's a bug report, new feature, correction, or additional documentation, we greatly value and appreciate your efforts to improve this project.
+
+Below are the guidelines for contributing to the BRRO Compressor. Please take a moment to review them before submitting your contributions.
+
+## Table of Contents
+- [Code of Conduct](#code-of-conduct)
+- [How to Contribute](#how-to-contribute)
+ - [Reporting Issues](#reporting-issues)
+ - [Submitting Code Changes](#submitting-code-changes)
+ - [Coding Standards](#coding-standards)
+- [Running Tests](#running-tests)
+- [Documentation](#documentation)
+
+## Code of Conduct
+
+This project adheres to a code of conduct that is based on respect, collaboration, and inclusivity. By participating in this project, you agree to uphold these values. Please report any issues or behavior that violates these guidelines to the project maintainers.
+
+## How to Contribute
+
+### Reporting Issues
+
+If you encounter a bug, performance issue, or have a suggestion for an enhancement, please open an issue in our [GitHub repository](https://github.com/instaclustr/atsc/issues). Make sure to provide as much detail as possible to help us understand the problem or suggestion. **But check for existing issues first!**
+
+When reporting an issue, please include:
+- A clear and descriptive title.
+- A detailed description of the issue or suggestion.
+- Steps to reproduce the issue, if applicable.
+- Any relevant logs, screenshots, or output that might help us diagnose the problem.
+
+### Submitting Code Changes
+
+We accept contributions via Pull Requests (PRs). Before submitting a PR, please follow these steps:
+
+1. **Fork the repository** to your GitHub account.
+2. **Create a new branch** for your changes.
+ - If you're creating a bugfix branch, name it XXX-something, where XXX is the issue number.
+ - If you're working on a new feature, first create an issue to announce your intentions, then name the branch XXX-something, where XXX is the issue number.
+3. **Make your changes** in this branch.
+4. **Test your changes** to ensure that they do not break existing functionality.
+5. **Commit your changes** with a meaningful commit message.
+6. **Push your changes** to your forked repository.
+7. **Create a Pull Request** to the main branch of the original repository.
+
+#### How to Create a Pull Request (PR)
+
+1. Go to the [GitHub repository](https://github.com/instaclustr/atsc).
+2. Click on the "Pull Requests" tab.
+3. Press the **"New pull request"** button.
+4. Select the **base repository** and the branch where you want to merge your changes (`main`).
+5. Compare this with the branch containing your changes (your forked repository and branch).
+6. Provide a **descriptive title** and **clear description** of the changes in the PR.
+7. Link any relevant issues by adding `Closes #IssueNumber` to the description.
+8. Submit your pull request.
+
+#### Guidelines for Pull Requests
+
+- Provide a clear description of your changes and the problem they solve.
+- Reference any relevant issues or discussions in your PR description.
+- When you make a PR for small change (such as fixing a typo, grammar fix), please squash your commits so that we can maintain a cleaner git history.
+- Include any necessary documentation updates as part of your PR.
+- Ensure that your code adheres to the project's coding standards (see below).
+- Code review comments might be added to your pull request. Please discuss them, implement the suggested changes, and push additional commits to your feature branch. Make sure to leave a comment after pushing.
+The new commits will appear in the pull request automatically, but the reviewers won't be notified unless you comment.
+- If your PR is a work-in-progress (WIP), please indicate this in the title by prefixing it with `[WIP]`.
+
+If your pull request isn't accepted right away, don't get discouraged! If there are issues with the implementation, you'll likely receive feedback on what can be improved.
+
+### Coding Standards
+
+To maintain consistency and quality across the codebase, please adhere to the following coding standards:
+
+- Follow the naming conventions and coding style already present in the codebase.
+- Keep functions and methods small and focused.
+- Write descriptive comments where necessary, especially in complex sections of code.
+- Ensure your code is linted and formatted according to the project's style guides.
+
+## Running Tests
+
+All code changes should be covered by tests. To run the test suite, use the following command: `cargo test`
+
+## Documentation
+
+Good documentation is crucial for the usability and maintainability of the project. If you are adding a new feature or modifying existing behavior, ensure that you update the relevant documentation files.
+
+- See `docs/README.md` for more information on building the docs and how docs get released.
+- Ensure that your changes are reflected in the usage instructions or other relevant sections.
+
+## Roadmap
+
+We have a long-term vision for the BRRO Compressor project, and we encourage contributors to get involved with features that are planned for future releases. Below is a brief overview of our current roadmap:
+
+- Implement streaming compression/decompression.
+- Expand frames to allow new data to be appended to existing frames.
+
+Feel free to contribute to these ongoing efforts or propose your own enhancements!
+
+## Feedback and Suggestions
+
+We value your feedback and suggestions! If you have any ideas on how to improve the BRRO Compressor, please don't hesitate to share them. You can do so by:
+
+- Opening a new issue labeled as a `suggestion` in our [GitHub repository](https://github.com/instaclustr/atsc).
+
+Your input helps us shape the future of the project, and we appreciate any suggestions or feedback you can provide.
diff --git a/Cargo.lock b/Cargo.lock
index 784ff38..fd48ec2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,21 +2,6 @@
# It is not intended for manual editing.
version = 3
-[[package]]
-name = "addr2line"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
-dependencies = [
- "gimli",
-]
-
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
[[package]]
name = "ahash"
version = "0.7.8"
@@ -107,12 +92,6 @@ dependencies = [
"windows-sys",
]
-[[package]]
-name = "anyhow"
-version = "1.0.86"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
-
[[package]]
name = "arrayvec"
version = "0.7.4"
@@ -120,14 +99,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
-name = "async-trait"
-version = "0.1.81"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
+name = "atsc"
+version = "0.5.0"
dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.72",
+ "average",
+ "bincode",
+ "clap",
+ "criterion",
+ "csv",
+ "env_logger",
+ "hound",
+ "inverse_distance_weight",
+ "log",
+ "median",
+ "num-traits",
+ "rand 0.8.5",
+ "regex",
+ "rustfft",
+ "splines",
+ "tempfile",
+ "thiserror",
+ "wavbrro",
]
[[package]]
@@ -147,27 +139,6 @@ dependencies = [
"num-traits",
]
-[[package]]
-name = "backtrace"
-version = "0.3.73"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a"
-dependencies = [
- "addr2line",
- "cc",
- "cfg-if",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
-]
-
-[[package]]
-name = "base64"
-version = "0.21.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
-
[[package]]
name = "bincode"
version = "2.0.0-rc.3"
@@ -211,37 +182,6 @@ dependencies = [
"wyz",
]
-[[package]]
-name = "block-buffer"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "brro-compressor"
-version = "0.5.0"
-dependencies = [
- "average",
- "bincode",
- "clap",
- "criterion",
- "env_logger 0.11.5",
- "hound",
- "inverse_distance_weight",
- "log",
- "median",
- "num-traits",
- "rand 0.8.5",
- "regex",
- "rustfft",
- "splines",
- "tempfile",
- "wavbrro",
-]
-
[[package]]
name = "bumpalo"
version = "3.16.0"
@@ -375,10 +315,10 @@ version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn 2.0.87",
]
[[package]]
@@ -405,15 +345,6 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
-[[package]]
-name = "cpufeatures"
-version = "0.2.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "criterion"
version = "0.5.1"
@@ -481,21 +412,11 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
-[[package]]
-name = "crypto-common"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
[[package]]
name = "csv"
-version = "1.3.0"
+version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
+checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"
dependencies = [
"csv-core",
"itoa",
@@ -507,10 +428,10 @@ dependencies = [
name = "csv-compressor"
version = "0.1.0"
dependencies = [
- "brro-compressor",
+ "atsc",
"clap",
"csv",
- "env_logger 0.11.5",
+ "env_logger",
"log",
"serde",
"tempdir",
@@ -527,22 +448,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "data-encoding"
-version = "2.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
-
-[[package]]
-name = "digest"
-version = "0.10.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
-dependencies = [
- "block-buffer",
- "crypto-common",
-]
-
[[package]]
name = "dtw_rs"
version = "0.9.5"
@@ -583,19 +488,6 @@ dependencies = [
"regex",
]
-[[package]]
-name = "env_logger"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580"
-dependencies = [
- "humantime",
- "is-terminal",
- "log",
- "regex",
- "termcolor",
-]
-
[[package]]
name = "env_logger"
version = "0.11.5"
@@ -609,12 +501,6 @@ dependencies = [
"log",
]
-[[package]]
-name = "equivalent"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
-
[[package]]
name = "errno"
version = "0.3.9"
@@ -637,33 +523,12 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
-[[package]]
-name = "fixedbitset"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
-
[[package]]
name = "float-ord"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d"
-[[package]]
-name = "fnv"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
-
-[[package]]
-name = "form_urlencoded"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
-dependencies = [
- "percent-encoding",
-]
-
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
@@ -676,95 +541,6 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
-[[package]]
-name = "futures"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
-dependencies = [
- "futures-channel",
- "futures-core",
- "futures-executor",
- "futures-io",
- "futures-sink",
- "futures-task",
- "futures-util",
-]
-
-[[package]]
-name = "futures-channel"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
-dependencies = [
- "futures-core",
- "futures-sink",
-]
-
-[[package]]
-name = "futures-core"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
-
-[[package]]
-name = "futures-executor"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
-dependencies = [
- "futures-core",
- "futures-task",
- "futures-util",
-]
-
-[[package]]
-name = "futures-io"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
-
-[[package]]
-name = "futures-macro"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.72",
-]
-
-[[package]]
-name = "futures-sink"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
-
-[[package]]
-name = "futures-task"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
-
-[[package]]
-name = "futures-util"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
-dependencies = [
- "futures-channel",
- "futures-core",
- "futures-io",
- "futures-macro",
- "futures-sink",
- "futures-task",
- "memchr",
- "pin-project-lite",
- "pin-utils",
- "slab",
-]
-
[[package]]
name = "generic-array"
version = "0.14.7"
@@ -786,31 +562,6 @@ dependencies = [
"wasi",
]
-[[package]]
-name = "gimli"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
-
-[[package]]
-name = "h2"
-version = "0.3.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
-dependencies = [
- "bytes",
- "fnv",
- "futures-core",
- "futures-sink",
- "futures-util",
- "http 0.2.12",
- "indexmap",
- "slab",
- "tokio",
- "tokio-util",
- "tracing",
-]
-
[[package]]
name = "half"
version = "2.4.1"
@@ -830,42 +581,6 @@ dependencies = [
"ahash",
]
-[[package]]
-name = "hashbrown"
-version = "0.14.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
-
-[[package]]
-name = "headers"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270"
-dependencies = [
- "base64",
- "bytes",
- "headers-core",
- "http 0.2.12",
- "httpdate",
- "mime",
- "sha1",
-]
-
-[[package]]
-name = "headers-core"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429"
-dependencies = [
- "http 0.2.12",
-]
-
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
[[package]]
name = "heck"
version = "0.5.0"
@@ -878,96 +593,18 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
-[[package]]
-name = "home"
-version = "0.5.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
-dependencies = [
- "windows-sys",
-]
-
[[package]]
name = "hound"
version = "3.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
-[[package]]
-name = "http"
-version = "0.2.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
-dependencies = [
- "bytes",
- "fnv",
- "itoa",
-]
-
-[[package]]
-name = "http"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
-dependencies = [
- "bytes",
- "fnv",
- "itoa",
-]
-
-[[package]]
-name = "http-body"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
-dependencies = [
- "bytes",
- "http 0.2.12",
- "pin-project-lite",
-]
-
-[[package]]
-name = "httparse"
-version = "1.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"
-
-[[package]]
-name = "httpdate"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
-
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
-[[package]]
-name = "hyper"
-version = "0.14.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9"
-dependencies = [
- "bytes",
- "futures-channel",
- "futures-core",
- "futures-util",
- "h2",
- "http 0.2.12",
- "http-body",
- "httparse",
- "httpdate",
- "itoa",
- "pin-project-lite",
- "socket2",
- "tokio",
- "tower-service",
- "tracing",
- "want",
-]
-
[[package]]
name = "iana-time-zone"
version = "0.1.60"
@@ -991,26 +628,6 @@ dependencies = [
"cc",
]
-[[package]]
-name = "idna"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
-dependencies = [
- "unicode-bidi",
- "unicode-normalization",
-]
-
-[[package]]
-name = "indexmap"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0"
-dependencies = [
- "equivalent",
- "hashbrown 0.14.5",
-]
-
[[package]]
name = "inverse_distance_weight"
version = "0.1.1"
@@ -1085,16 +702,6 @@ version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
-[[package]]
-name = "lock_api"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
-dependencies = [
- "autocfg",
- "scopeguard",
-]
-
[[package]]
name = "log"
version = "0.4.22"
@@ -1107,209 +714,54 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "993f81eac178937d7d54dc7ba7bddd70a526bca92b36d95016add46f59d19754"
dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "memchr"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
-
-[[package]]
-name = "mime"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
-
-[[package]]
-name = "mime_guess"
-version = "2.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
-dependencies = [
- "mime",
- "unicase",
-]
-
-[[package]]
-name = "miniz_oxide"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
-dependencies = [
- "adler",
-]
-
-[[package]]
-name = "mio"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
-dependencies = [
- "hermit-abi",
- "libc",
- "wasi",
- "windows-sys",
-]
-
-[[package]]
-name = "multer"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2"
-dependencies = [
- "bytes",
- "encoding_rs",
- "futures-util",
- "http 0.2.12",
- "httparse",
- "log",
- "memchr",
- "mime",
- "spin",
- "version_check",
-]
-
-[[package]]
-name = "multimap"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
-
-[[package]]
-name = "num-complex"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "num-integer"
-version = "0.1.46"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
-dependencies = [
- "autocfg",
- "libm",
-]
-
-[[package]]
-name = "object"
-version = "0.36.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
-
-[[package]]
-name = "oorandom"
-version = "11.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
-
-[[package]]
-name = "optimizer"
-version = "0.1.0"
-dependencies = [
- "chrono",
- "clap",
- "claxon",
- "env_logger 0.11.5",
- "hound",
- "log",
- "median",
- "regex",
-]
-
-[[package]]
-name = "parking_lot"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-targets",
+ "generic-array",
]
[[package]]
-name = "percent-encoding"
-version = "2.3.1"
+name = "memchr"
+version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
-name = "petgraph"
-version = "0.6.5"
+name = "num-complex"
+version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
- "fixedbitset",
- "indexmap",
+ "num-traits",
]
[[package]]
-name = "pin-project"
-version = "1.1.5"
+name = "num-integer"
+version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
dependencies = [
- "pin-project-internal",
+ "num-traits",
]
[[package]]
-name = "pin-project-internal"
-version = "1.1.5"
+name = "num-traits"
+version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.72",
+ "autocfg",
+ "libm",
]
[[package]]
-name = "pin-project-lite"
-version = "0.2.14"
+name = "once_cell"
+version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
-name = "pin-utils"
-version = "0.1.0"
+name = "oorandom"
+version = "11.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
[[package]]
name = "plotters"
@@ -1348,16 +800,6 @@ dependencies = [
"zerocopy",
]
-[[package]]
-name = "prettyplease"
-version = "0.1.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86"
-dependencies = [
- "proc-macro2",
- "syn 1.0.109",
-]
-
[[package]]
name = "primal-check"
version = "0.3.4"
@@ -1376,96 +818,6 @@ dependencies = [
"unicode-ident",
]
-[[package]]
-name = "prom-remote-api"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a91e313f40839f01a242d526f12aff757b69d9c50f59c10947b58f50fad45e4"
-dependencies = [
- "async-trait",
- "bytes",
- "env_logger 0.10.2",
- "futures",
- "prost",
- "prost-build",
- "snap",
- "warp",
-]
-
-[[package]]
-name = "prometheus-remote"
-version = "0.1.0"
-dependencies = [
- "async-trait",
- "chrono",
- "clap",
- "claxon",
- "dtw_rs",
- "env_logger 0.11.5",
- "hound",
- "log",
- "median",
- "prom-remote-api",
- "regex",
- "symphonia",
- "tokio",
- "warp",
-]
-
-[[package]]
-name = "prost"
-version = "0.11.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
-dependencies = [
- "bytes",
- "prost-derive",
-]
-
-[[package]]
-name = "prost-build"
-version = "0.11.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270"
-dependencies = [
- "bytes",
- "heck 0.4.1",
- "itertools",
- "lazy_static",
- "log",
- "multimap",
- "petgraph",
- "prettyplease",
- "prost",
- "prost-types",
- "regex",
- "syn 1.0.109",
- "tempfile",
- "which",
-]
-
-[[package]]
-name = "prost-derive"
-version = "0.11.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
-dependencies = [
- "anyhow",
- "itertools",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "prost-types"
-version = "0.11.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
-dependencies = [
- "prost",
-]
-
[[package]]
name = "ptr_meta"
version = "0.1.4"
@@ -1588,15 +940,6 @@ dependencies = [
"rand_core 0.3.1",
]
-[[package]]
-name = "redox_syscall"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
-dependencies = [
- "bitflags 2.6.0",
-]
-
[[package]]
name = "regex"
version = "1.10.5"
@@ -1653,7 +996,7 @@ dependencies = [
"bitvec",
"bytecheck",
"bytes",
- "hashbrown 0.12.3",
+ "hashbrown",
"ptr_meta",
"rend",
"rkyv_derive",
@@ -1673,12 +1016,6 @@ dependencies = [
"syn 1.0.109",
]
-[[package]]
-name = "rustc-demangle"
-version = "0.1.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
-
[[package]]
name = "rustfft"
version = "6.2.0"
@@ -1722,18 +1059,6 @@ dependencies = [
"winapi-util",
]
-[[package]]
-name = "scoped-tls"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
-
-[[package]]
-name = "scopeguard"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-
[[package]]
name = "seahash"
version = "4.1.0"
@@ -1757,7 +1082,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn 2.0.87",
]
[[package]]
@@ -1772,81 +1097,12 @@ dependencies = [
"serde",
]
-[[package]]
-name = "serde_urlencoded"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
-dependencies = [
- "form_urlencoded",
- "itoa",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "sha1"
-version = "0.10.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
-
-[[package]]
-name = "signal-hook-registry"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "simdutf8"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
-[[package]]
-name = "slab"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
-
-[[package]]
-name = "snap"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
-
-[[package]]
-name = "socket2"
-version = "0.5.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
-dependencies = [
- "libc",
- "windows-sys",
-]
-
-[[package]]
-name = "spin"
-version = "0.9.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
-
[[package]]
name = "splines"
version = "4.3.1"
@@ -2011,9 +1267,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.72"
+version = "2.0.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d"
dependencies = [
"proc-macro2",
"quote",
@@ -2048,33 +1304,24 @@ dependencies = [
"windows-sys",
]
-[[package]]
-name = "termcolor"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
-dependencies = [
- "winapi-util",
-]
-
[[package]]
name = "thiserror"
-version = "1.0.63"
+version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
+checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.63"
+version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
+checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn 2.0.87",
]
[[package]]
@@ -2102,60 +1349,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
-[[package]]
-name = "tokio"
-version = "1.39.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
-dependencies = [
- "backtrace",
- "bytes",
- "libc",
- "mio",
- "parking_lot",
- "pin-project-lite",
- "signal-hook-registry",
- "socket2",
- "tokio-macros",
- "windows-sys",
-]
-
-[[package]]
-name = "tokio-macros"
-version = "2.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.72",
-]
-
-[[package]]
-name = "tokio-tungstenite"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38"
-dependencies = [
- "futures-util",
- "log",
- "tokio",
- "tungstenite",
-]
-
-[[package]]
-name = "tokio-util"
-version = "0.7.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"
-dependencies = [
- "bytes",
- "futures-core",
- "futures-sink",
- "pin-project-lite",
- "tokio",
-]
-
[[package]]
name = "tools"
version = "0.1.0"
@@ -2164,7 +1357,7 @@ dependencies = [
"clap",
"claxon",
"dtw_rs",
- "env_logger 0.11.5",
+ "env_logger",
"hound",
"log",
"median",
@@ -2173,32 +1366,6 @@ dependencies = [
"wavbrro",
]
-[[package]]
-name = "tower-service"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
-
-[[package]]
-name = "tracing"
-version = "0.1.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
-dependencies = [
- "log",
- "pin-project-lite",
- "tracing-core",
-]
-
-[[package]]
-name = "tracing-core"
-version = "0.1.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
-dependencies = [
- "once_cell",
-]
-
[[package]]
name = "transpose"
version = "0.2.3"
@@ -2209,84 +1376,18 @@ dependencies = [
"strength_reduce",
]
-[[package]]
-name = "try-lock"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
-
-[[package]]
-name = "tungstenite"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1"
-dependencies = [
- "byteorder",
- "bytes",
- "data-encoding",
- "http 1.1.0",
- "httparse",
- "log",
- "rand 0.8.5",
- "sha1",
- "thiserror",
- "url",
- "utf-8",
-]
-
[[package]]
name = "typenum"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
-[[package]]
-name = "unicase"
-version = "2.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
-dependencies = [
- "version_check",
-]
-
-[[package]]
-name = "unicode-bidi"
-version = "0.3.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
-
[[package]]
name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
-[[package]]
-name = "unicode-normalization"
-version = "0.1.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
-dependencies = [
- "tinyvec",
-]
-
-[[package]]
-name = "url"
-version = "2.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
-dependencies = [
- "form_urlencoded",
- "idna",
- "percent-encoding",
-]
-
-[[package]]
-name = "utf-8"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
-
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -2329,44 +1430,6 @@ dependencies = [
"winapi-util",
]
-[[package]]
-name = "want"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
-dependencies = [
- "try-lock",
-]
-
-[[package]]
-name = "warp"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c"
-dependencies = [
- "bytes",
- "futures-channel",
- "futures-util",
- "headers",
- "http 0.2.12",
- "hyper",
- "log",
- "mime",
- "mime_guess",
- "multer",
- "percent-encoding",
- "pin-project",
- "scoped-tls",
- "serde",
- "serde_json",
- "serde_urlencoded",
- "tokio",
- "tokio-tungstenite",
- "tokio-util",
- "tower-service",
- "tracing",
-]
-
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
@@ -2394,7 +1457,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn 2.0.87",
"wasm-bindgen-shared",
]
@@ -2416,7 +1479,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn 2.0.87",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -2431,7 +1494,7 @@ checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
name = "wavbrro"
version = "0.1.0"
dependencies = [
- "env_logger 0.11.5",
+ "env_logger",
"log",
"rkyv",
"tempfile",
@@ -2447,18 +1510,6 @@ dependencies = [
"wasm-bindgen",
]
-[[package]]
-name = "which"
-version = "4.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
-dependencies = [
- "either",
- "home",
- "once_cell",
- "rustix",
-]
-
[[package]]
name = "winapi"
version = "0.3.9"
@@ -2599,5 +1650,5 @@ checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn 2.0.87",
]
diff --git a/Cargo.toml b/Cargo.toml
index 5f43db9..06e5a54 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,8 +1,6 @@
[workspace]
members = [
- "brro-compressor",
- "optimizer",
- "prometheus-remote",
+ "atsc",
"tools",
"wavbrro",
"vsri",
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..8783c22
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2024 NetApp, Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
\ No newline at end of file
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
new file mode 100644
index 0000000..940fabc
--- /dev/null
+++ b/MAINTAINERS.md
@@ -0,0 +1,11 @@
+# ATSC Maintainers
+
+## Maintainers
+
+| Maintainer | GitHub ID | Affiliation |
+|------------------|-------------------------------------------------|-------------------------------------------------------|
+| Carlos Rolo | [cjrolo](https://github.com/cjrolo) | [Instaclustr](https://www.github.com/instaclustr/) |
+| Lucas Kent | [rukai](https://github.com/rukai) | [Instaclustr](https://www.github.com/instaclustr/) |
+| Bohdan Siryk | [worryg0d](https://github.com/worryg0d) | [Instaclustr](https://www.github.com/instaclustr/) |
+| Danylo Savchenko | [testisnullus](https://github.com/testisnullus) | [Instaclustr](https://www.github.com/instaclustr/) |
+| Conor | [conorbros](https://github.com/conorbros) | [Instaclustr](https://www.github.com/instaclustr/) |
\ No newline at end of file
diff --git a/README.md b/README.md
index eba30bd..d11b8d8 100644
--- a/README.md
+++ b/README.md
@@ -1,87 +1,111 @@
-# BRRO Compressor
+# ATSC - Advance Time Series Compressor
-Version: 0.5 Released: 30/11/2023
+## Documentation
-## Major Changes
+For full documentation please go to [Docs](https://github.com/instaclustr/atsc/tree/main/docs)
-### 0.5
+## Building ATSC
-- Added Polynomial Compressor (with 2 variants)
-- Created and Integrated a proper file type (wbro)
-- Benchmarks of the different compressors
-- Integration testing
-- Several fixes and cleanups
+1. Clone the repository:
-## Description
+ ```bash
+ git clone https://github.com/instaclustr/atsc
+ cd atsc
+ ```
-BRRO Compressor is a compressor that relies on the characteristics of a signal to provide a far greater compression that currently existing ones. BRRO relies on different techniques based on a initial analysis of the signal to use the best suited method for compressor for that specific signal segment.
+2. Build the project:
-For a detailed description on the compressor methods and logic check `BRRO.md`.
+ ```bash
+ cargo build --release
+ ```
-## Usage
+## What is ATSC?
-Currently BRRO relies on have Raw BRRO files generated by our prometheus remote endpoint. This would work as input for the compressor.
+Advanced Time Series Compressor (in short: ATSC), is a configurable, *lossy* compressor that uses the characteristics of a time-series to create a function approximation of the time series.
+This way, ATSC only needs to store the parametrization of the function and not the data.
+ATSC draws inspiration from established compression and signal analysis techniques, achieving significant compression ratios.
+In internal testing ATSC compressed from 46 times to 880 times the time series of our databases with a fitting error within 1% of the original time-series.
+In some cases, ATSC would produce highly compressed data without any data loss (Perfect fitting functions).
+ATSC is meant to be used in long term storage of time series, as it benefits from more points to do a better fitting.
+The decompression of data is faster (up to 40x) vs a slower compression speed, as it is expected that the data might be compressed once and decompressed several times.
-Compressor usage:
+Internally ATSC uses the following methods for time series fitting:
-```An Advanced Time-Series Compressor
+* FFT (Fast Fourier Transforms)
+* Constant
+* Interpolation - Catmull-Rom
+* Interpolation - Inverse Distance Weight
-Usage: brro-compressor [OPTIONS]
+For a more detailed insight into ATSC read the paper here: [ATSC - A novel approach to time-series compression](https://some.url.com)
-Arguments:
- input file
+Currently, ATSC uses an internal format to process time series (WBRO) and outputs a compressed format (BRO). A CSV to WBRO format is available here: [CSV Compressor](https://github.com/instaclustr/atsc/tree/main/csv-compressor)
-Options:
- --compressor [default: auto] [possible values: auto, noop, fft, wavelet, constant, polynomial, top-bottom]
- -u Uncompresses the input file/directory
- -h, --help Print help
- -V, --version Print version
- ```
+## Where does ATSC fits?
-## Programs and description
+ATSC fits in any place that needs space reduction in trade for precision.
+ATSC is to time series what JPG/MP3 is to image/audio.
+If there is no need of absolute precision of the output vs the original input, you could probably use ATSC.
-This repository contains one main program and other programs that serve different purposes, some are for just testing, others do some actual work.
+Example of use cases:
-### flac-server
+* In places where time series are rolled over, ATSC is a perfect fit. It would probably offer more space savings without any meaningful loss in precision.
+* Time series that are under sampled (e.g. once every 20sec). With ATSC you can greatly increase sample rate (e.g. once per second) without losing space.
+* Long, slow moving data series (e.g. Weather data). Those will most probably follow an easy to fit pattern
+* Data that is meant to be visualized by humans and not machine processed (e.g. Operation teams). With such a small error, under 1%, it shouldn't impact analysis.
-**NOTE**: Remote read is currently NOT working, as it depends on FLAC files that are no longer generated.
+## Usage ATSC
-Needs a prometheus server. We need it to get our samples out. Supports read and write from prometheus.
+### Prerequisites
-Launch the `flac-server` and set it as your remote endpoint for prometheus, example below.
+* Ensure you have [Rust](https://www.rust-lang.org/tools/install) and Cargo installed on your system.
-```YAML
-# Remote read and Write
-remote_write:
- - url: "http://localhost:9201/api/write"
+### Usage
-remote_read:
- - url: "http://localhost:9201/api/read"
- read_recent: true
- name: "flac_server"
-```
+ATSC relies on files with a WBRO extension to operate, learn more about that here: [WBRO - A time series format](https://github.com/instaclustr/atsc/tree/main/wavbrro)
+You can also compress from CSV with the provided [CSV tool](https://github.com/instaclustr/atsc/tree/main/csv-compressor)
+Those files would work as input for the compressor.
-Make Prometheus server a source of your grafana and check the data.
+Compressor usage:
-### brro_optimizer
+```bash
+Usage: atsc [OPTIONS]
-Maybe the most important tool at this point, it picks a WAV file from the datasets described below and optimizes it into a way that we might see a meaningful compression into FLAC.
-The tool also has options to dump the output of the file as a single sample per period, instead of the 4 channels. This is good to obtain the data as it was feed into the flac-server.
-The code performs optimizations based on file name, so renaming might cause issues.
+Arguments:
+ input file
-Usage (Getting raw samples): `./brro_optimizer infile.wav --dump-raw > file.raw`
-Usage (Getting optimized samples): `./brro_optimizer infile.wav --dump-optimized > file.raw`
-Usage (Generate a optimized file): `./brro_optimizer -w infile.wav`
+ --compressor
+ Select a compressor, default is auto [default: auto] [possible values: auto, noop, fft, constant, polynomial, idw]
+ -e, --error
+ Sets the maximum allowed error for the compressed data, must be between 0 and 50. Default is 5 (5%). 0 is lossless compression 50 will do a median filter on the data. In between will pick optimize for the error [default: 5]
+ -u
+ Uncompresses the input file/directory
+ -c, --compression-selection-sample-level
+ Samples the input data instead of using all the data for selecting the optimal compressor. Only impacts speed, might or not increased compression ratio. For best results use 0 (default). Only works when compression = Auto. 0 will use all the data (slowest) 6 will sample 128 data points (fastest) [default: 0]
+ --verbose
+ Verbose output, dumps everysample in the input file (for compression) and in the ouput file (for decompression)
+ -h, --help
+ Print help
+ -V, --version
+ Print version
+```
-If you set the ENV Variable for Debug it will output what it is doing.
+#### Compress a File
-### Matlab folder
+To compress a file using ATSC, run:
-Exploratory code. Should be removed.
+```bash
+atsc
+```
+
+### Decompress a File
+To decompress a file, use:
+```bash
+atsc -u
+```
## Roadmap
-1. Update `flac-server` to read/write WBRO/BRO files.
-2. Streaming compression/decompression
-3. Automated compressor selection
-4. Frame expansion (Allowing new data to be appended to existing frames)
+* Frame expansion (Allowing new data to be appended to existing frames)
+* Dynamic function loading (e.g. providing more functions without touching the whole code base)
+* Global/Per frame error storage
+* Efficient error encoding
\ No newline at end of file
diff --git a/brro-compressor/Cargo.toml b/atsc/Cargo.toml
similarity index 92%
rename from brro-compressor/Cargo.toml
rename to atsc/Cargo.toml
index 548f79b..c9bb4c8 100644
--- a/brro-compressor/Cargo.toml
+++ b/atsc/Cargo.toml
@@ -1,5 +1,5 @@
[package]
-name = "brro-compressor"
+name = "atsc"
version = "0.5.0"
authors = ["Carlos Rolo "]
edition = "2021"
@@ -22,6 +22,8 @@ wavbrro = { path = "../wavbrro" }
splines = "4.3.0"
inverse_distance_weight = "0.1.1"
num-traits = "0.2"
+csv = "1.3.1"
+thiserror = "2.0.3"
[dev-dependencies]
criterion = "0.5.1"
@@ -33,4 +35,4 @@ harness= false
[[bench]]
name = "polynomial_bench"
-harness= false
\ No newline at end of file
+harness= false
diff --git a/brro-compressor/benches/fft_bench.rs b/atsc/benches/fft_bench.rs
similarity index 67%
rename from brro-compressor/benches/fft_bench.rs
rename to atsc/benches/fft_bench.rs
index 0153178..ccaf8fe 100644
--- a/brro-compressor/benches/fft_bench.rs
+++ b/atsc/benches/fft_bench.rs
@@ -1,9 +1,47 @@
-use brro_compressor::compressor::fft::{fft, fft_allowed_error, fft_set, fft_to_data, FFT};
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use atsc::compressor::fft::{fft, fft_allowed_error, fft_set, fft_to_data, FFT};
use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use std::path::PathBuf;
+use wavbrro::wavbrro::WavBrro;
+
+const TEST_WBRO_PATH: &str = "tests/wbros/go_gc_heap_goal_bytes.wbro";
-// Basic FFT compression benchmark
+/// Loads the file, and returns the Vec result
+fn load_data_from_wbro_file() -> Vec {
+ let test_file_path = PathBuf::from(TEST_WBRO_PATH);
+ WavBrro::from_file(&test_file_path).unwrap()
+}
+
+fn pad_to_size(mut data: Vec, desired_size: usize) -> Vec {
+ // Check if the data length is smaller than the desired size
+ if data.len() < desired_size {
+ // Add zeroes to the data until it reaches the desired size
+ data.resize(desired_size, 1.0);
+ } else {
+ // Truncate the data to the exact size if it's larger
+ data.truncate(desired_size);
+ }
+ data
+}
+
+/// Basic FFT compression benchmark
fn fft_basic_benchmark(c: &mut Criterion) {
- let data = vec![1.0; 1024];
+ let data = load_data_from_wbro_file();
c.bench_function("FFT Compression (Basic)", |b| {
b.iter(|| {
@@ -15,7 +53,7 @@ fn fft_basic_benchmark(c: &mut Criterion) {
// Advanced FFT compression benchmark with custom frequency set
fn fft_advanced_benchmark(c: &mut Criterion) {
- let data = vec![1.0; 1024];
+ let data = load_data_from_wbro_file();
c.bench_function("FFT Compression (Advanced)", |b| {
b.iter(|| {
@@ -27,7 +65,7 @@ fn fft_advanced_benchmark(c: &mut Criterion) {
// Error-constrained FFT compression benchmark
fn fft_error_benchmark(c: &mut Criterion) {
- let data = vec![1.0; 1024];
+ let data = load_data_from_wbro_file();
let max_error = 0.01;
c.bench_function("FFT Compression (Error Constrained)", |b| {
@@ -40,7 +78,7 @@ fn fft_error_benchmark(c: &mut Criterion) {
// FFT decompression benchmark
fn fft_decompression_benchmark(c: &mut Criterion) {
- let data = vec![1.0; 1024];
+ let data = load_data_from_wbro_file();
let compressed_data = fft(&data);
c.bench_function("FFT Decompression", |b| {
@@ -62,7 +100,7 @@ fn fft_initialization_benchmark(c: &mut Criterion) {
// Decompression without FFT benchmark
fn decompression_without_fft_benchmark(c: &mut Criterion) {
- let data = vec![1.0; 1024];
+ let data = load_data_from_wbro_file();
let compressed_data = fft(&data);
c.bench_function("Decompression without FFT", |b| {
@@ -75,9 +113,11 @@ fn decompression_without_fft_benchmark(c: &mut Criterion) {
// FFT compression benchmark with varying data sizes
fn fft_varying_data_size_benchmark(c: &mut Criterion) {
- let data_small = vec![1.0; 256];
- let data_medium = vec![1.0; 1024];
- let data_large = vec![1.0; 4096];
+ let data = load_data_from_wbro_file();
+
+ let data_small = pad_to_size(data.clone(), 256);
+ let data_medium = pad_to_size(data.clone(), 1024);
+ let data_large = pad_to_size(data, 4096);
c.bench_function("FFT Compression (Small Data)", |b| {
b.iter(|| {
@@ -100,7 +140,7 @@ fn fft_varying_data_size_benchmark(c: &mut Criterion) {
// Compression ratio vs. time benchmark
fn compression_ratio_vs_time_benchmark(c: &mut Criterion) {
- let data = vec![1.0; 1024];
+ let data = load_data_from_wbro_file();
c.bench_function("Compression Ratio vs. Compression Time", |b| {
b.iter(|| {
@@ -115,7 +155,7 @@ fn compression_ratio_vs_time_benchmark(c: &mut Criterion) {
// Multiple compression rounds benchmark
fn multiple_compression_rounds_benchmark(c: &mut Criterion) {
- let data = vec![1.0; 1024];
+ let data = load_data_from_wbro_file();
let rounds = 10;
c.bench_function("Multiple Compression Rounds", |b| {
diff --git a/brro-compressor/benches/polynomial_bench.rs b/atsc/benches/polynomial_bench.rs
similarity index 91%
rename from brro-compressor/benches/polynomial_bench.rs
rename to atsc/benches/polynomial_bench.rs
index 01087c9..e5648dc 100644
--- a/brro-compressor/benches/polynomial_bench.rs
+++ b/atsc/benches/polynomial_bench.rs
@@ -1,7 +1,21 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
// Import necessary libraries
-use brro_compressor::compressor::polynomial::{
- polynomial, polynomial_allowed_error, to_data, PolynomialType,
-};
+use atsc::compressor::polynomial::{polynomial, polynomial_allowed_error, to_data, PolynomialType};
use criterion::{black_box, criterion_group, criterion_main, Criterion};
// Define example sample data
diff --git a/brro-compressor/demo/run_demo.sh b/atsc/demo/run_demo.sh
similarity index 63%
rename from brro-compressor/demo/run_demo.sh
rename to atsc/demo/run_demo.sh
index f19796f..d8358c6 100755
--- a/brro-compressor/demo/run_demo.sh
+++ b/atsc/demo/run_demo.sh
@@ -11,37 +11,37 @@ do
cp $infilename tmp.wbro
- ../../target/debug/brro-compressor --compressor fft --error $i --verbose tmp.wbro > $mfile
+ ../../target/debug/atsc --compressor fft --error $i --verbose tmp.wbro > $mfile
echo "FFT Size: "
du -sb tmp.bro
- ../../target/debug/brro-compressor -u --verbose tmp.bro >> $mfile
+ ../../target/debug/atsc -u --verbose tmp.bro >> $mfile
sed -i -e 's/Output/output_fft/g' $mfile
cp $infilename tmp.wbro
- ../../target/debug/brro-compressor --compressor idw --error $i tmp.wbro > /dev/null
+ ../../target/debug/atsc --compressor idw --error $i tmp.wbro > /dev/null
echo "IDW Size: "
du -sb tmp.bro
- ../../target/debug/brro-compressor -u --verbose tmp.bro >> $mfile
+ ../../target/debug/atsc -u --verbose tmp.bro >> $mfile
sed -i -e 's/Output/output_idw/g' $mfile
cp $infilename tmp.wbro
- ../../target/debug/brro-compressor --compressor polynomial --error $i tmp.wbro > /dev/null
+ ../../target/debug/atsc --compressor polynomial --error $i tmp.wbro > /dev/null
echo "Polynomial Size: "
du -sb tmp.bro
- ../../target/debug/brro-compressor -u --verbose tmp.bro >> $mfile
+ ../../target/debug/atsc -u --verbose tmp.bro >> $mfile
sed -i -e 's/Output/output_poly/g' $mfile
cp $infilename tmp.wbro
- ../../target/debug/brro-compressor --error $i tmp.wbro > /dev/null
+ ../../target/debug/atsc --error $i tmp.wbro > /dev/null
echo "Auto Size: "
du -sb tmp.bro
- ../../target/debug/brro-compressor -u --verbose tmp.bro >> $mfile
+ ../../target/debug/atsc -u --verbose tmp.bro >> $mfile
sed -i -e 's/Output/output_auto/g' $mfile
diff --git a/brro-compressor/src/compressor/constant.rs b/atsc/src/compressor/constant.rs
similarity index 89%
rename from brro-compressor/src/compressor/constant.rs
rename to atsc/src/compressor/constant.rs
index 4c399ff..cc82e21 100644
--- a/brro-compressor/src/compressor/constant.rs
+++ b/atsc/src/compressor/constant.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use crate::{
compressor::CompressorResult,
optimizer::utils::{Bitdepth, DataStats},
@@ -53,7 +69,6 @@ impl Decode for Constant {
) -> Result {
let id = Decode::decode(decoder)?;
let bitdepth = Decode::decode(decoder)?;
- // Here is where the pig twists the tail
let constant: f64 = match bitdepth {
Bitdepth::U8 => {
debug!("Decoding as u8");
@@ -96,11 +111,6 @@ impl Constant {
}
}
- /// This compressor is about having a single constant for the whole segment
- pub fn set_constant(&mut self, constant_value: f64) {
- self.constant = constant_value;
- }
-
/// Receives a data stream and generates a Constant
pub fn decompress(data: &[u8]) -> Self {
let config = BinConfig::get();
@@ -110,7 +120,6 @@ impl Constant {
/// This function transforms the structure into a Binary stream
pub fn to_bytes(&self) -> Vec {
- // Use Bincode and flate2-rs? Do this at the Stream Level?
let config = BinConfig::get();
bincode::encode_to_vec(self, config).unwrap()
}
@@ -125,9 +134,7 @@ impl Constant {
pub fn constant_compressor(data: &[f64], stats: DataStats) -> CompressorResult {
debug!("Initializing Constant Compressor. Error and Stats provided");
- // Initialize the compressor
let c = Constant::new(data.len(), stats.min, stats.bitdepth);
- // Convert to bytes
CompressorResult::new(c.to_bytes(), 0.0)
}
diff --git a/brro-compressor/src/compressor/fft.rs b/atsc/src/compressor/fft.rs
similarity index 84%
rename from brro-compressor/src/compressor/fft.rs
rename to atsc/src/compressor/fft.rs
index a195138..4c9fa07 100644
--- a/brro-compressor/src/compressor/fft.rs
+++ b/atsc/src/compressor/fft.rs
@@ -1,4 +1,23 @@
-use crate::{optimizer::utils::DataStats, utils::error::calculate_error};
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use crate::{
+ optimizer::utils::DataStats,
+ utils::{error::calculate_error, next_size},
+};
use bincode::{Decode, Encode};
use rustfft::{num_complex::Complex, FftPlanner};
use std::{cmp::Ordering, collections::BinaryHeap};
@@ -20,30 +39,6 @@ pub struct FrequencyPoint {
}
impl FrequencyPoint {
- pub fn new(real: f32, img: f32) -> Self {
- FrequencyPoint {
- pos: 0,
- freq_real: real,
- freq_img: img,
- }
- }
-
- pub fn with_position(real: f32, img: f32, pos: u16) -> Self {
- FrequencyPoint {
- pos,
- freq_real: real,
- freq_img: img,
- }
- }
-
- pub fn from_complex(complex: Complex) -> Self {
- FrequencyPoint {
- pos: 0,
- freq_real: complex.re,
- freq_img: complex.im,
- }
- }
-
pub fn from_complex_with_position(complex: Complex, pos: u16) -> Self {
FrequencyPoint {
pos,
@@ -113,15 +108,10 @@ impl Ord for FrequencyPoint {
/// FFT Compressor. Applies FFT to a signal, picks the N best frequencies, discards the rest. Always LOSSY
#[derive(PartialEq, Debug)]
pub struct FFT {
- /// Compressor ID
pub id: u8,
- /// Stored frequencies
pub frequencies: Vec,
- /// The maximum numeric value of the points in the frame
pub max_value: f32,
- /// The minimum numeric value of the points in the frame
pub min_value: f32,
- /// Compression error
pub error: Option,
}
@@ -189,6 +179,30 @@ impl FFT {
y
}
+ /// Given an array of size N, it returns the next best FFT size with the
+ /// begining and the ended padded to improve Gibbs on the edges of the frame
+ pub fn gibbs_sizing(data: &[f64]) -> Vec {
+ let data_len = data.len();
+ let next_size = next_size(data_len);
+ let added_len = next_size - data_len;
+ debug!("Gibbs sizing, padding with {}", added_len);
+ let prefix_len = added_len / 2;
+ let suffix_len = added_len - prefix_len;
+ // Extend the beginning and the end with the first and last value
+ let mut result = Vec::with_capacity(next_size);
+ if let Some(first) = data.first() {
+ result.resize(prefix_len, *first);
+ result.extend_from_slice(data);
+
+ if let Some(last) = data.last() {
+ for _ in 0..suffix_len {
+ result.push(*last);
+ }
+ }
+ }
+ result
+ }
+
/// Rounds a number to the specified number of decimal places
// TODO: Move this into utils? I think this will be helpfull somewhere else.
fn round(&self, x: f32, decimals: u32) -> f64 {
@@ -267,7 +281,7 @@ impl FFT {
self.frequencies = FFT::fft_trim(&mut buffer, max_freq);
}
- /// Compress data via FFT - VERY EXPENSIVE
+ /// Compress data via FFT - EXPENSIVE
/// This picks a set of data, computes the FFT, and optimizes the number of frequencies to store to match
/// the max allowed error.
/// NOTE: This does not otimize for smallest possible error, just being smaller than the error.
@@ -276,16 +290,29 @@ impl FFT {
debug!("Same max and min, we're done here!");
return;
}
- // Variables
- let len = data.len();
- let len_f32 = len as f32;
- if !len.is_power_of_two() {
+
+ if !data.len().is_power_of_two() {
warn!("Slow FFT, data segment is not a power of 2!");
}
// Let's start from the defaults values for frequencies
- let max_freq = if 3 >= (len / 100) { 3 } else { len / 100 };
+ let max_freq = if 3 >= (data.len() / 100) {
+ 3
+ } else {
+ data.len() / 100
+ };
+
+ // Should we apply a Gibbs sizing?
+ let g_data: &[f64] = if data.len() >= 128 {
+ &FFT::gibbs_sizing(data)
+ } else {
+ data
+ };
+
+ let len = g_data.len();
+ let len_f32 = len as f32;
+
// Clean the data
- let mut buffer = FFT::optimize(data);
+ let mut buffer = FFT::optimize(g_data);
// Create the FFT planners
let mut planner = FftPlanner::new();
@@ -315,7 +342,7 @@ impl FFT {
.iter()
.map(|&f| self.round(f.re / len_f32, DECIMAL_PRECISION.into()))
.collect();
- current_err = calculate_error(data, &out_data);
+ current_err = calculate_error(g_data, &out_data);
trace!("Current Err: {}", current_err);
// Max iterations is 22 (We start at 10%, we can go to 95% and 1% at a time)
match iterations {
@@ -335,7 +362,7 @@ impl FFT {
}
/// Compresses data via FFT
- /// The set of frequencies to store is 1/100 of the data lenght OR 3, which is bigger.
+ /// The set of frequencies to store is 1/100 of the data length OR 3, which is bigger.
pub fn compress(&mut self, data: &[f64]) {
if self.max_value == self.min_value {
debug!("Same max and min, we're done here!");
@@ -359,8 +386,6 @@ impl FFT {
buffer.truncate(size);
self.frequencies = FFT::fft_trim(&mut buffer, max_freq);
}
-
- /// Decompresses data
pub fn decompress(data: &[u8]) -> Self {
let config = BinConfig::get();
let (fft, _) = bincode::decode_from_slice(data, config).unwrap();
@@ -403,21 +428,37 @@ impl FFT {
debug!("Same max and min, faster decompression!");
return vec![self.max_value as f64; frame_size];
}
+ // Was this processed to reduce the Gibbs phenomeon?
+ let trim_sizes = if frame_size >= 128 {
+ let added_len = next_size(frame_size) - frame_size;
+ let prefix_len = added_len / 2;
+ let suffix_len = added_len - prefix_len;
+ debug!(
+ "Gibbs sizing detected, removing padding with {} len",
+ added_len
+ );
+ (prefix_len, suffix_len)
+ } else {
+ (0, 0)
+ };
+ let gibbs_frame_size = frame_size + trim_sizes.0 + trim_sizes.1;
// Vec to process the ifft
- let mut data = self.get_mirrored_freqs(frame_size);
+ let mut data = self.get_mirrored_freqs(gibbs_frame_size);
// Plan the ifft
let mut planner = FftPlanner::new();
- let fft = planner.plan_fft_inverse(frame_size);
+ let fft = planner.plan_fft_inverse(gibbs_frame_size);
// run the ifft
fft.process(&mut data);
// We need this for normalization
- let len = frame_size as f32;
+ let len = gibbs_frame_size as f32;
// We only need the real part
- let out_data = data
- .iter()
+ data.iter()
+ // trim the exceses data
+ .skip(trim_sizes.0)
+ .take(data.len() - trim_sizes.0 - trim_sizes.1)
+ // We only need the real part
.map(|&f| self.round(f.re / len, DECIMAL_PRECISION.into()))
- .collect();
- out_data
+ .collect()
}
}
@@ -547,6 +588,17 @@ mod tests {
assert!(e <= 0.01);
}
+ #[test]
+ fn test_gibbs_sizing() {
+ let mut vector1 = vec![2.0; 2048];
+ vector1[0] = 1.0;
+ vector1[2047] = 3.0;
+ let vector1_sized = FFT::gibbs_sizing(&vector1);
+ assert_eq!(vector1_sized.len(), 2187);
+ assert!(vector1_sized[2] == 1.0);
+ assert!(vector1_sized[2185] == 3.0);
+ }
+
#[test]
fn test_static_and_trim() {
// This vector should lead to 11 frequencies
diff --git a/brro-compressor/src/compressor/mod.rs b/atsc/src/compressor/mod.rs
similarity index 86%
rename from brro-compressor/src/compressor/mod.rs
rename to atsc/src/compressor/mod.rs
index 475f838..75f0370 100644
--- a/brro-compressor/src/compressor/mod.rs
+++ b/atsc/src/compressor/mod.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use bincode::config::{self, Configuration};
use bincode::{Decode, Encode};
diff --git a/brro-compressor/src/compressor/noop.rs b/atsc/src/compressor/noop.rs
similarity index 82%
rename from brro-compressor/src/compressor/noop.rs
rename to atsc/src/compressor/noop.rs
index 512d8ed..75b9d63 100644
--- a/brro-compressor/src/compressor/noop.rs
+++ b/atsc/src/compressor/noop.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use super::BinConfig;
use bincode::{Decode, Encode};
use log::{debug, info};
@@ -18,17 +34,14 @@ impl Noop {
data: Vec::with_capacity(sample_count),
}
}
- ///Optimize
pub fn optimize(data: &[f64]) -> Vec {
let mut out_vec = Vec::with_capacity(data.len());
for &element in data {
- // Round the floating-point number before casting to i64
out_vec.push(element.round() as i64);
}
out_vec
}
- /// "Compress"
pub fn compress(&mut self, data: &[f64]) {
self.data = Noop::optimize(data);
debug!(
@@ -51,7 +64,6 @@ impl Noop {
bincode::encode_to_vec(self, config).unwrap()
}
- /// Returns an array of data
pub fn to_data(&self, _frame_size: usize) -> Vec {
self.data.clone()
}
@@ -102,7 +114,7 @@ mod tests {
fn test_optimize() {
// Test case with floating-point numbers that have fractional parts
let input_data = [1.5, 2.7, 3.3, 4.9];
- let expected_output = [2, 3, 3, 5]; // Rounded to the nearest integer
+ let expected_output = [2, 3, 3, 5];
let result = Noop::optimize(&input_data);
assert_eq!(result, expected_output);
diff --git a/brro-compressor/src/compressor/polynomial.rs b/atsc/src/compressor/polynomial.rs
similarity index 96%
rename from brro-compressor/src/compressor/polynomial.rs
rename to atsc/src/compressor/polynomial.rs
index 7f6a2a8..531a82a 100644
--- a/brro-compressor/src/compressor/polynomial.rs
+++ b/atsc/src/compressor/polynomial.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use crate::optimizer::utils::{Bitdepth, DataStats};
use crate::utils::{error::calculate_error, round_and_limit_f64, round_f64, DECIMAL_PRECISION};
@@ -26,17 +42,12 @@ pub enum Method {
#[derive(PartialEq, Debug, Clone)]
pub struct Polynomial {
- /// Compressor ID
pub id: PolynomialType,
- /// Stored Points
pub data_points: Vec,
pub min: f64,
pub max: f64,
- /// What is the base step between points
pub point_step: u8,
- /// Compression error
pub error: Option,
- /// Target bitdepth
pub bitdepth: Bitdepth,
}
@@ -81,7 +92,6 @@ impl Decode for Polynomial {
) -> Result {
let id = Decode::decode(decoder)?;
let bitdepth = Decode::decode(decoder)?;
- // Here is where the pig twists the tail
let data_points: Vec = match bitdepth {
Bitdepth::U8 => {
debug!("Decoding as u8");
@@ -189,10 +199,6 @@ impl Polynomial {
}
}
- fn locate_in_data_points(&self, point: f64) -> bool {
- self.data_points.iter().any(|&i| i == point)
- }
-
fn get_method(&self) -> Method {
match self.id {
PolynomialType::Idw => Method::Idw,
@@ -207,7 +213,7 @@ impl Polynomial {
}
// TODO: Big one, read below
// To reduce error we add more points to the polynomial, but, we also might add residuals
- // each residual is 1/data_lenght * 100% less compression, each jump is 5% less compression.
+ // each residual is 1/data_length * 100% less compression, each jump is 5% less compression.
// We can do the math and pick the one which fits better.
let method = self.get_method();
let data_len = data.len();
@@ -264,7 +270,7 @@ impl Polynomial {
}
self.error = Some(current_err);
debug!(
- "Final Stored Data Lenght: {} Iterations: {}",
+ "Final Stored Data Length: {} Iterations: {}",
self.data_points.len(),
iterations
);
@@ -298,7 +304,6 @@ impl Polynomial {
self.point_step = step as u8;
}
- // --- MANDATORY METHODS ---
pub fn compress(&mut self, data: &[f64]) {
let points = if 3 >= (data.len() / 100) {
3
@@ -308,7 +313,6 @@ impl Polynomial {
self.compress_hinted(data, points)
}
- /// Decompresses data
pub fn decompress(data: &[u8]) -> Self {
let config = BinConfig::get();
let (poly, _) = bincode::decode_from_slice(data, config).unwrap();
@@ -320,7 +324,6 @@ impl Polynomial {
bincode::encode_to_vec(self, config).unwrap()
}
- // --- END OF MANDATORY METHODS ---
/// Since IDW and Polynomial are the same code everywhere, this function prepares the data
/// to be used by one of the polynomial decompression methods
fn get_positions(&self, frame_size: usize) -> Vec {
@@ -377,7 +380,6 @@ impl Polynomial {
.map(|&f| f as f64)
.collect();
let idw = IDW::new(points, self.data_points.clone());
- // Build the data
(0..frame_size)
.map(|f| {
round_and_limit_f64(
@@ -405,11 +407,8 @@ impl Polynomial {
pub fn polynomial(data: &[f64], p_type: PolynomialType) -> Vec {
info!("Initializing Polynomial Compressor");
let stats = DataStats::new(data);
- // Initialize the compressor
let mut c = Polynomial::new(data.len(), stats.min, stats.max, p_type, stats.bitdepth);
- // Convert the data
c.compress(data);
- // Convert to bytes
c.to_bytes()
}
@@ -420,14 +419,11 @@ pub fn polynomial_allowed_error(
) -> CompressorResult {
info!("Initializing Polynomial Compressor");
let stats = DataStats::new(data);
- // Initialize the compressor
let mut c = Polynomial::new(data.len(), stats.min, stats.max, p_type, stats.bitdepth);
- // Convert the data
c.compress_bounded(data, allowed_error);
CompressorResult::new(c.to_bytes(), c.error.unwrap_or(0.0))
}
-/// Uncompress
pub fn to_data(sample_number: usize, compressed_data: &[u8]) -> Vec {
let c = Polynomial::decompress(compressed_data);
c.to_data(sample_number)
diff --git a/atsc/src/csv.rs b/atsc/src/csv.rs
new file mode 100644
index 0000000..eb08d9b
--- /dev/null
+++ b/atsc/src/csv.rs
@@ -0,0 +1,252 @@
+use std::fs::{File, OpenOptions};
+use std::path::Path;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("Failed to open csv file")]
+ OpenFileFailed,
+
+ #[error("Timestamp filed is not found")]
+ TimestampFieldNotFound,
+
+ #[error("Value field is not found")]
+ ValueFieldNotFound,
+
+ #[error("Parsing timestamp is failed")]
+ ParsingTimestampFailed,
+
+ #[error("Parsing value is failed")]
+ ParsingValueFailed,
+
+ #[error("Unexpected error occurred")]
+ Unexpected,
+}
+
+type Result = std::result::Result;
+
+#[derive(Debug, PartialEq)]
+pub struct Sample {
+ pub timestamp: i64,
+ pub value: f64,
+}
+
+/// read_samples_with_headers reads samples from the given file.
+/// It expects that timestamps are stored under timestamp_field field header
+/// and values are stored under value_field.
+pub fn read_samples_with_headers(
+ filepath: &Path,
+ timestamp_field: &str,
+ value_field: &str,
+) -> Result> {
+ let mut reader = open_csv_reader(filepath, true)?;
+ let headers = reader.headers().map_err(|_| Error::Unexpected)?;
+
+ // Find the index of the timestamp and value fields
+ let _ = headers
+ .iter()
+ .position(|h| h == timestamp_field)
+ .ok_or(Error::TimestampFieldNotFound)?;
+
+ let value_idx = headers
+ .iter()
+ .position(|h| h == value_field)
+ .ok_or(Error::ValueFieldNotFound)?;
+
+ // Collect samples
+ let mut samples = Vec::new();
+ for record in reader.records() {
+ let record = record.unwrap();
+ // let timestamp: DateTime = record.get(timestamp_idx).unwrap().parse()?;
+ let value: f64 = record
+ .get(value_idx)
+ .unwrap()
+ .parse()
+ .map_err(|_| Error::ParsingValueFailed)?;
+
+ samples.push(Sample {
+ timestamp: 0,
+ value,
+ });
+ }
+
+ Ok(samples)
+}
+
+/// read_samples reads samples from the given file.
+/// It assumes that file contains no headers and
+/// consists of only a single field with values.
+pub fn read_samples(filepath: &Path) -> Result> {
+ let mut reader = open_csv_reader(filepath, false)?;
+
+ // Collect samples
+ let mut samples = Vec::new();
+ for record in reader.records() {
+ let record = record.unwrap();
+ let value: f64 = record
+ .get(0) // assuming that there is only a single field with values inside
+ .unwrap()
+ .parse()
+ .map_err(|_| Error::ParsingValueFailed)?;
+
+ samples.push(Sample {
+ timestamp: 0,
+ value,
+ });
+ }
+
+ Ok(samples)
+}
+
+fn open_csv_reader(filepath: &Path, has_headers: bool) -> Result> {
+ let file = OpenOptions::new()
+ .read(true)
+ .open(filepath)
+ .map_err(|_| Error::OpenFileFailed)?;
+
+ let reader = csv::ReaderBuilder::new()
+ .has_headers(has_headers)
+ .from_reader(file);
+ Ok(reader)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::fs::File;
+ use std::io::Write;
+ use std::path::Path;
+ use tempfile::tempdir;
+
+ fn create_csv_file(content: &str, filepath: &Path) {
+ let mut file = File::create(filepath).expect("Failed to create test CSV file");
+ file.write_all(content.as_bytes())
+ .expect("Failed to write to test CSV file");
+ }
+
+ #[test]
+ fn test_valid_csv() {
+ let temp_dir = tempdir().unwrap();
+ let filepath = temp_dir.into_path().join("test_valid.csv");
+
+ let content = "timestamp,value\n1625097600,123.45\n1625184000,678.90\n";
+ create_csv_file(content, &filepath);
+
+ let result = read_samples_with_headers(&filepath, "timestamp", "value");
+ assert!(result.is_ok());
+
+ let samples = result.unwrap();
+ assert_eq!(
+ samples,
+ vec![
+ Sample {
+ timestamp: 0,
+ value: 123.45
+ },
+ Sample {
+ timestamp: 0,
+ value: 678.90
+ },
+ ]
+ );
+ }
+
+ #[test]
+ fn test_single_column_csv_no_headers() {
+ let temp_dir = tempdir().unwrap();
+ let filepath = temp_dir.into_path().join("test_single_column.csv");
+
+ let content = "123.45\n678.90\n";
+ create_csv_file(content, &filepath);
+
+ let result = read_samples(&filepath);
+ assert!(result.is_ok());
+
+ let samples = result.unwrap();
+ assert_eq!(
+ samples,
+ vec![
+ Sample {
+ timestamp: 0,
+ value: 123.45
+ },
+ Sample {
+ timestamp: 0,
+ value: 678.90
+ },
+ ]
+ );
+ }
+
+ #[test]
+ fn test_incorrect_format_single_column() {
+ let temp_dir = tempdir().unwrap();
+ let filepath = temp_dir.into_path().join("test_incorrect_format.csv");
+
+ let content = "value\n123.45\ninvalid_value\n678.90\n";
+ create_csv_file(content, &filepath);
+
+ let result = read_samples(&filepath);
+ assert!(matches!(result, Err(Error::ParsingValueFailed)));
+ }
+
+ #[test]
+ fn test_missing_timestamp_column() {
+ let temp_dir = tempdir().unwrap();
+ let filepath = temp_dir
+ .into_path()
+ .join("test_missing_timestamp_column.csv");
+
+ let content = "time,value\n1625097600,123.45\n1625184000,678.90\n";
+ create_csv_file(content, &filepath);
+
+ let result = read_samples_with_headers(&filepath, "timestamp", "value");
+ assert!(matches!(result, Err(Error::TimestampFieldNotFound)));
+ }
+
+ #[test]
+ fn test_missing_value_column() {
+ let temp_dir = tempdir().unwrap();
+ let filepath = temp_dir.into_path().join("test_missing_value_column.csv");
+
+ let content = "timestamp,price\n1625097600,123.45\n1625184000,678.90\n";
+ create_csv_file(content, &filepath);
+
+ let result = read_samples_with_headers(&filepath, "timestamp", "value");
+ assert!(matches!(result, Err(Error::ValueFieldNotFound)));
+ }
+
+ #[test]
+ fn test_parsing_error_value() {
+ let temp_dir = tempdir().unwrap();
+ let filepath = temp_dir.into_path().join("test_parsing_error_value.csv");
+
+ let content = "timestamp,value\n1625097600,invalid_value\n1625184000,678.90\n";
+ create_csv_file(content, &filepath);
+
+ let result = read_samples_with_headers(&filepath, "timestamp", "value");
+ assert!(matches!(result, Err(Error::ParsingValueFailed)));
+ }
+
+ #[test]
+ fn test_unopenable_file() {
+ let filepath = Path::new("/invalid/path/to/non_existent_file.csv");
+
+ let result = read_samples_with_headers(filepath, "timestamp", "value");
+ assert!(matches!(result, Err(Error::OpenFileFailed)));
+
+ let result = read_samples(filepath);
+ assert!(matches!(result, Err(Error::OpenFileFailed)));
+ }
+
+ #[test]
+ fn test_no_headers_csv() {
+ let temp_dir = tempdir().unwrap();
+ let filepath = temp_dir.into_path().join("test_no_headers.csv");
+
+ let content = "timestamp,value\n1625097600,123.45\n1625184000,678.90\n";
+ create_csv_file(content, &filepath);
+
+ let result = read_samples(&filepath);
+ assert!(matches!(result, Err(Error::ParsingValueFailed)));
+ }
+}
diff --git a/brro-compressor/src/data.rs b/atsc/src/data.rs
similarity index 88%
rename from brro-compressor/src/data.rs
rename to atsc/src/data.rs
index 164ba6b..ea97dd9 100644
--- a/brro-compressor/src/data.rs
+++ b/atsc/src/data.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use crate::compressor::{BinConfig, Compressor};
use crate::frame::CompressorFrame;
use crate::header::CompressorHeader;
@@ -11,7 +27,6 @@ pub struct CompressedStream {
}
impl CompressedStream {
- /// Creates an empty compressor stream
pub fn new() -> Self {
CompressedStream {
header: CompressorHeader::new(),
@@ -71,8 +86,6 @@ impl CompressedStream {
let (compressed_stream, _) = bincode::decode_from_slice(data, config).unwrap();
compressed_stream
}
-
- /// Decompresses all the frames and returns a vector with the data
pub fn decompress(&self) -> Vec {
self.data_frames
.iter()
diff --git a/brro-compressor/src/frame/mod.rs b/atsc/src/frame/mod.rs
similarity index 70%
rename from brro-compressor/src/frame/mod.rs
rename to atsc/src/frame/mod.rs
index 08a5dc2..0f031e6 100644
--- a/brro-compressor/src/frame/mod.rs
+++ b/atsc/src/frame/mod.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use crate::{compressor::Compressor, optimizer::utils::DataStats};
use bincode::{Decode, Encode};
use log::debug;
@@ -10,9 +26,7 @@ const COMPRESSION_SPEED: [i32; 7] = [i32::MAX, 4096, 2048, 1024, 512, 256, 128];
pub struct CompressorFrame {
/// The frame size in bytes,
frame_size: usize,
- /// The number of samples in this frame,
sample_count: usize,
- /// The compressor used in the current frame
compressor: Compressor,
/// Output from the compressor
data: Vec,
@@ -32,7 +46,7 @@ impl CompressorFrame {
}
/// Calculates the size of the Frame and "closes it"
- // TODO this is probably wrong, so we have to use the write stream to dump the bytes writen
+ // TODO this is probably wrong, so we have to use the write stream to dump the bytes written
pub fn close(&mut self) {
let size = size_of_val(&self.sample_count)
+ size_of_val(&self.compressor)
@@ -60,11 +74,7 @@ impl CompressorFrame {
// We need enough samples to do decent compression, minimum is 128 (2^7)
let data_sample = COMPRESSION_SPEED[compression_speed] as usize;
// Eligible compressors for use
- let compressor_list = [
- Compressor::Constant,
- Compressor::FFT,
- Compressor::Polynomial,
- ];
+ let compressor_list = [Compressor::FFT, Compressor::Polynomial];
// Do a statistical analysis of the data, let's see if we can pick a compressor out of this.
let stats = DataStats::new(data);
// Checking the statistical analysis and chose, if possible, a compressor
@@ -90,6 +100,7 @@ impl CompressorFrame {
compressor,
)
})
+ .filter(|(result, _)| result.error <= max_error as f64)
.min_by_key(|x| x.0.compressed_data.len())
.unwrap();
self.compressor = *chosen_compressor;
@@ -100,19 +111,39 @@ impl CompressorFrame {
.compressed_data;
} else {
// Run all the eligible compressors and choose smallest
- let (smallest_result, chosen_compressor) = compressor_list
+ let compressor_results: Vec<_> = compressor_list
.iter()
.map(|compressor| {
(
compressor.get_compress_bounded_results(data, max_error as f64),
- compressor,
+ *compressor,
)
})
- .min_by_key(|x| x.0.compressed_data.len())
- .unwrap();
+ .collect();
- self.data = smallest_result.compressed_data;
- self.compressor = *chosen_compressor;
+ #[allow(
+ clippy::neg_cmp_op_on_partial_ord,
+ reason = "we need to exactly negate `result.error < max_error`, we can't apply de morgans to the expression due to NaN values"
+ )]
+ let best_compressor = if compressor_results
+ .iter()
+ .all(|(result, _)| !(result.error <= max_error as f64))
+ {
+ // To ensure we always have at least one result,
+ // if all results are above the max error just pick the smallest.
+ compressor_results
+ .into_iter()
+ .min_by_key(|x| x.0.compressed_data.len())
+ } else {
+ compressor_results
+ .into_iter()
+ .filter(|(result, _)| result.error <= max_error as f64)
+ .min_by_key(|x| x.0.compressed_data.len())
+ };
+
+ let (result, compressor) = best_compressor.unwrap();
+ self.data = result.compressed_data;
+ self.compressor = compressor;
}
debug!("Auto Compressor Selection: {:?}", self.compressor);
}
diff --git a/atsc/src/header.rs b/atsc/src/header.rs
new file mode 100644
index 0000000..080beb0
--- /dev/null
+++ b/atsc/src/header.rs
@@ -0,0 +1,38 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use bincode::{Decode, Encode};
+
+#[derive(Encode, Decode, Debug, Clone)]
+pub struct CompressorHeader {
+ initial_segment: [u8; 4],
+ // We should go unsigned
+ frame_count: i16,
+}
+
+impl CompressorHeader {
+ pub fn new() -> Self {
+ CompressorHeader {
+ initial_segment: *b"BRRO",
+ // We have to limit the bytes of the header
+ frame_count: 0,
+ }
+ }
+
+ pub fn add_frame(&mut self) {
+ self.frame_count += 1;
+ }
+}
diff --git a/atsc/src/lib.rs b/atsc/src/lib.rs
new file mode 100644
index 0000000..83b5a3c
--- /dev/null
+++ b/atsc/src/lib.rs
@@ -0,0 +1,29 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#![allow(clippy::new_without_default)]
+// TODO: re-enable dead code checks
+#![allow(dead_code)]
+extern crate core;
+
+pub mod compressor;
+pub mod data;
+pub mod frame;
+pub mod header;
+pub mod utils;
+
+pub mod csv;
+pub mod optimizer;
diff --git a/brro-compressor/src/main.rs b/atsc/src/main.rs
similarity index 73%
rename from brro-compressor/src/main.rs
rename to atsc/src/main.rs
index 97a2738..671a816 100644
--- a/brro-compressor/src/main.rs
+++ b/atsc/src/main.rs
@@ -1,7 +1,24 @@
-use brro_compressor::compressor::Compressor;
-use brro_compressor::data::CompressedStream;
-use brro_compressor::optimizer::OptimizerPlan;
-use brro_compressor::utils::readers::bro_reader;
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use atsc::compressor::Compressor;
+use atsc::csv::{read_samples, read_samples_with_headers};
+use atsc::data::CompressedStream;
+use atsc::optimizer::OptimizerPlan;
+use atsc::utils::readers::bro_reader;
use clap::{arg, command, Parser};
use log::{debug, error};
use std::error::Error;
@@ -30,7 +47,6 @@ fn process_args(arguments: &Args) -> Result<(), Box> {
Ok(())
}
-/// Processes all files in a given directory.
fn process_directory(arguments: &Args) -> Result<(), Box> {
// Assuming you want to process each file inside this directory
for entry in std::fs::read_dir(arguments.input.clone())? {
@@ -51,7 +67,6 @@ fn process_directory(arguments: &Args) -> Result<(), Box> {
Ok(())
}
-/// Processes a single file.
fn process_single_file(mut file_path: PathBuf, arguments: &Args) -> Result<(), Box> {
debug!("Processing single file...");
if arguments.uncompress {
@@ -66,12 +81,41 @@ fn process_single_file(mut file_path: PathBuf, arguments: &Args) -> Result<(), B
file_path.set_extension("wbro");
WavBrro::to_file_with_data(&file_path, &decompressed_data)
}
+ } else if arguments.csv {
+ // Read samples from csv and compress it
+ let samples = if arguments.no_header {
+ debug!("Reading samples from csv with no header");
+ read_samples(&file_path)?
+ } else {
+ debug!("Reading samples from csv with headers");
+ let headers_option = match arguments.fields.clone() {
+ Some(fields) => fields,
+ None => "time, value".to_string(),
+ };
+ let headers: Vec<_> = headers_option.split(",").collect();
+ // Assuming that header[0] is a time field and header[1] is value field
+ read_samples_with_headers(&file_path, headers[0], headers[1])?
+ };
+
+ let data: Vec = samples.into_iter().map(|sample| sample.value).collect();
+
+ if arguments.verbose {
+ println!("Input={:?}", data);
+ }
+
+ // Compress
+ let compressed_data = compress_data(&data, arguments);
+
+ // Write
+ file_path.set_extension("bro");
+ std::fs::write(file_path, compressed_data)?;
} else {
// Read an WavBRRO file and compress it
let data = WavBrro::from_file(&file_path)?;
if arguments.verbose {
println!("Input={:?}", data);
}
+
//compress
let compressed_data = compress_data(&data, arguments);
@@ -127,7 +171,6 @@ fn decompress_data(compressed_data: &[u8]) -> Vec {
#[derive(Parser, Default, Debug)]
#[command(author, version, about="A Time-Series compressor", long_about = None)]
struct Args {
- /// input file
input: PathBuf,
/// Select a compressor, default is auto
@@ -156,6 +199,21 @@ struct Args {
/// Verbose output, dumps everysample in the input file (for compression) and in the ouput file (for decompression)
#[arg(long, action)]
verbose: bool,
+
+ /// Defines user input as a CSV file
+ #[arg(long, action)]
+ csv: bool,
+
+ /// Defines if the CSV has no header
+ #[arg(long, action)]
+ no_header: bool,
+
+ /// Defines names of fields in CSV file. It should follow this format:
+ /// --fields=TIME_FIELD_NAME,VALUE_FIELD_NAME
+ /// It assumes that the one before comma is a name of time field and the one
+ /// after comma is value field.
+ #[arg(long, default_value = "time,value")]
+ fields: Option,
}
#[derive(clap::ValueEnum, Default, Clone, Debug)]
diff --git a/brro-compressor/src/optimizer/mod.rs b/atsc/src/optimizer/mod.rs
similarity index 81%
rename from brro-compressor/src/optimizer/mod.rs
rename to atsc/src/optimizer/mod.rs
index dfcff96..0a0ded3 100644
--- a/brro-compressor/src/optimizer/mod.rs
+++ b/atsc/src/optimizer/mod.rs
@@ -1,17 +1,29 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use crate::{
compressor::Compressor,
- types,
utils::{f64_to_u64, prev_power_of_two},
};
-use log::debug;
-use types::metric_tag::MetricTag;
pub mod utils;
/// Max Frame size, this can aprox. 36h of data at 1point/sec rate, a little more than 1 week at 1point/5sec
/// and 1 month (30 days) at 1 point/20sec.
/// This would be aprox. 1MB of Raw data (131072 * 64bits).
-/// We wouldn't want to decompressed a ton of uncessary data, but for historical view of the data, looking into 1day/week/month at once is very reasonable
const MAX_FRAME_SIZE: usize = 131072; // 2^17
/// The Min frame size is one that allows our compressors potentially achieve 100x compression. Currently the most
/// limited one is the FFT compressor, that needs 3 frequencies at minimum, 3x100 = 300, next power of 2 is 512.
@@ -43,20 +55,6 @@ impl OptimizerPlan {
}
}
- /// Creates an optimal plan for compression for the data set provided bound by a given error
- pub fn plan_bounded(data: &[f64], max_error: f32) -> Self {
- // TODO: Check error limits
- let c_data = OptimizerPlan::clean_data(data);
- let chunks = OptimizerPlan::get_chunks_sizes(c_data.len());
- let optimizer = OptimizerPlan::assign_compressor(&c_data, &chunks, Some(max_error));
- OptimizerPlan {
- data: c_data,
- chunk_sizes: chunks,
- compressors: optimizer,
- }
- }
-
- /// Sets a given compressor for all data chunks
pub fn set_compressor(&mut self, compressor: Compressor) {
let new_compressors = vec![compressor; self.compressors.len()];
self.compressors = new_compressors;
@@ -111,14 +109,11 @@ impl OptimizerPlan {
}
/// Walks the data, checks how much variability is in the data, and assigns a compressor based on that
- /// NOTE: Is this any good?
fn get_compressor(data: &[f64]) -> Compressor {
let _ = data.iter().map(|&f| f64_to_u64(f, 0));
- // For now, let's just return FFT
Compressor::FFT
}
- /// Assigns a compressor to a chunk of data
fn assign_compressor(
clean_data: &[f64],
chunks: &[usize],
@@ -139,17 +134,6 @@ impl OptimizerPlan {
}
}
-/// This should look at the data and return an optimized dataset for a specific compressor,
-/// If a compressor is hand picked, this should be skipped.
-pub fn process_data(wav_data: &[f64], tag: &MetricTag) -> Vec {
- debug!("Tag: {:?} Len: {}", tag, wav_data.len());
- wav_data
- .iter()
- .filter(|x| !(x.is_nan() || x.is_infinite()))
- .copied()
- .collect()
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/brro-compressor/src/optimizer/utils.rs b/atsc/src/optimizer/utils.rs
similarity index 74%
rename from brro-compressor/src/optimizer/utils.rs
rename to atsc/src/optimizer/utils.rs
index b2a4532..0dbf73a 100644
--- a/brro-compressor/src/optimizer/utils.rs
+++ b/atsc/src/optimizer/utils.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use bincode::{Decode, Encode};
use log::debug;
@@ -10,17 +26,11 @@ pub enum Bitdepth {
}
/// Data structure that holds statictical information about the data provided
pub struct DataStats {
- // Max value
pub max: f64,
- // Max value location in the array
pub max_loc: usize,
- // Min value
pub min: f64,
- // Min value location in the array
pub min_loc: usize,
- // Mean of the data
pub mean: f64,
- // Bitdepth that this data can be
pub bitdepth: Bitdepth,
pub fractional: bool,
}
@@ -54,7 +64,7 @@ impl DataStats {
}
mean /= data.len() as f64;
// Check max size of values
- // For very large numbers (i32 and i64), it might be ideal to detect the dc component
+ // TODO: for very large numbers (i32 and i64), it might be ideal to detect the dc component
// of the signal. And then remove it later
let max_int = split_n(max).0; // This is the DC component
let min_int = split_n(min).0;
@@ -102,19 +112,6 @@ impl DataStats {
}
}
}
-
-fn as_i8(value: f64) -> i8 {
- split_n(value).0 as i8
-}
-
-fn as_i16(value: f64) -> i16 {
- split_n(value).0 as i16
-}
-
-fn as_i32(value: f64) -> i32 {
- split_n(value).0 as i32
-}
-
fn split_n(x: f64) -> (i64, f64) {
const FRACT_SCALE: f64 = 1.0 / (65536.0 * 65536.0 * 65536.0 * 65536.0); // 1_f64.exp(-64)
const STORED_MANTISSA_DIGITS: u32 = f64::MANTISSA_DIGITS - 1;
@@ -161,57 +158,6 @@ fn split_n(x: f64) -> (i64, f64) {
(0, 0.0)
}
}
-
-fn analyze_data(data: &Vec) -> (i32, i64, bool) {
- let mut min: f64 = 0.0;
- let mut max: f64 = 0.0;
- let mut fractional = false;
- for value in data {
- let t_value = *value;
- if split_n(t_value).1 != 0.0 {
- fractional = true;
- }
- if t_value > max {
- max = t_value
- };
- if t_value < min {
- min = t_value
- };
- }
- // Check max size of values
- // For very large numbers (i32 and i64), it might be ideal to detect the dc component
- // of the signal. And then remove it later
- let max_int = split_n(max).0; // This is the DC component
- let min_int = split_n(min).0;
-
- // Finding the bitdepth without the DC component
- let recommended_bitdepth = find_bitdepth(max_int - min_int, min_int);
- debug!(
- "Recommended Bitdepth: {}, Fractional: {}",
- recommended_bitdepth, fractional
- );
- (recommended_bitdepth, min_int, fractional)
-}
-
-fn find_bitdepth(max_int: i64, min_int: i64) -> i32 {
- // Check where those ints fall into
- let bitdepth = match max_int {
- _ if max_int <= u8::MAX.into() => 8,
- _ if max_int <= i16::MAX.into() => 16,
- _ if max_int <= i32::MAX.into() => 32,
- _ => 64,
- };
-
- let bitdepth_signed = match min_int {
- _ if min_int == 0 => 8,
- _ if min_int >= i16::MIN.into() => 16,
- _ if min_int >= i32::MIN.into() => 32,
- _ => 64,
- };
-
- bitdepth.max(bitdepth_signed)
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/brro-compressor/src/utils/error.rs b/atsc/src/utils/error.rs
similarity index 90%
rename from brro-compressor/src/utils/error.rs
rename to atsc/src/utils/error.rs
index 2c46797..d472b5d 100644
--- a/brro-compressor/src/utils/error.rs
+++ b/atsc/src/utils/error.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use std::cmp;
#[derive(Default, Debug, Clone)]
@@ -24,17 +40,11 @@ impl ErrorMethod {
/// This function calculates the error between 2 arrays of f64. The results are from 0 to ..
/// Being 0, no error, 1 - 100% error and so on.
-/// This uses the default function to calculte it.
+/// This uses the default function to calculate it.
pub fn calculate_error(original: &[f64], generated: &[f64]) -> f64 {
ErrorMethod::error(ErrorMethod::default(), original, generated)
}
-/// This function calculates the error between 2 arrays of f64. The results are from 0 to ..
-/// Being 0, no error, 1 - 100% error and so on.
-/// This uses the provided method to calculte it.
-pub fn calculate_error_method(original: &[f64], generated: &[f64], method: ErrorMethod) -> f64 {
- ErrorMethod::error(method, original, generated)
-}
/// Calculates the mean squared error between two vectors.
///
/// # Arguments
diff --git a/brro-compressor/src/utils/mod.rs b/atsc/src/utils/mod.rs
similarity index 50%
rename from brro-compressor/src/utils/mod.rs
rename to atsc/src/utils/mod.rs
index 255dd23..c00015d 100644
--- a/brro-compressor/src/utils/mod.rs
+++ b/atsc/src/utils/mod.rs
@@ -1,10 +1,26 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
pub mod error;
pub mod readers;
pub mod writers;
pub const DECIMAL_PRECISION: u32 = 5;
-// Is this the right place?
+// TODO: check if it is the right place?
pub fn prev_power_of_two(n: usize) -> usize {
// n = 0 gives highest_bit_set_idx = 0.
let highest_bit_set_idx = 63 - (n | 1).leading_zeros();
@@ -12,6 +28,26 @@ pub fn prev_power_of_two(n: usize) -> usize {
(1 << highest_bit_set_idx) & n
}
+/// Given a number N, checks what is the next number that is in the form of (2^N * 3^M)
+pub fn next_size(mut n: usize) -> usize {
+ n += 1;
+ while !is_decomposable(n) {
+ n += 1;
+ }
+ n
+}
+
+/// Checks if a number is in the form of (2^N * 3^M), usefull for FFT sizing
+pub fn is_decomposable(mut n: usize) -> bool {
+ while n % 2 == 0 {
+ n /= 2;
+ }
+ while n % 3 == 0 {
+ n /= 3;
+ }
+ n == 1
+}
+
/// Converts a float to u64 with a given precision
pub fn f64_to_u64(number: f64, precision: usize) -> u64 {
// TODO: Panic on overflow
@@ -22,11 +58,6 @@ pub fn f64_to_u64(number: f64, precision: usize) -> u64 {
(number * mul as f64) as u64
}
-pub fn round_f32(x: f32, decimals: u32) -> f64 {
- let y = 10i32.pow(decimals) as f64;
- (x as f64 * y).round() / y
-}
-
pub fn round_f64(x: f64, decimals: u32) -> f64 {
let y = 10i32.pow(decimals) as f64;
(x * y).round() / y
@@ -53,4 +84,19 @@ mod tests {
assert_eq!(round_and_limit_f64(1., 2., 4., 1), 2.0);
assert_eq!(round_and_limit_f64(3.123452312, 2., 4., 3), 3.123);
}
+
+ #[test]
+ fn test_is_decomposable() {
+ assert!(is_decomposable(2048));
+ assert!(is_decomposable(512));
+ }
+
+ #[test]
+ fn test_next_size() {
+ assert_eq!(next_size(2048), 2187);
+ assert_eq!(next_size(512), 576);
+ assert_eq!(next_size(256), 288);
+ assert_eq!(next_size(128), 144);
+ assert_eq!(next_size(12432), 13122);
+ }
}
diff --git a/brro-compressor/src/utils/readers/bro_reader.rs b/atsc/src/utils/readers/bro_reader.rs
similarity index 59%
rename from brro-compressor/src/utils/readers/bro_reader.rs
rename to atsc/src/utils/readers/bro_reader.rs
index 22047cf..f2b7e37 100644
--- a/brro-compressor/src/utils/readers/bro_reader.rs
+++ b/atsc/src/utils/readers/bro_reader.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
// Implement a streaming reader here
use std::fs;
use std::fs::File;
@@ -28,28 +44,3 @@ fn is_bro_file(file_path: &Path) -> io::Result {
// Check if the file starts with "BRRO"
Ok(header.starts_with(b"BRRO"))
}
-
-/// Read a file by chunks and processes the chunks
-pub fn process_by_chunk(file_path: &Path) -> Result<(), std::io::Error> {
- let mut file = std::fs::File::open(file_path)?;
-
- let mut list_of_chunks = Vec::new();
- // 64KB at a time, assuming 64Bit samples, ~1024 samples.
- let chunk_size = 0x10000;
-
- loop {
- let mut chunk = Vec::with_capacity(chunk_size);
- let n = file
- .by_ref()
- .take(chunk_size as u64)
- .read_to_end(&mut chunk)?;
- if n == 0 {
- break;
- }
- list_of_chunks.push(chunk);
- if n < chunk_size {
- break;
- }
- }
- Ok(())
-}
diff --git a/atsc/src/utils/readers/mod.rs b/atsc/src/utils/readers/mod.rs
new file mode 100644
index 0000000..b557bbc
--- /dev/null
+++ b/atsc/src/utils/readers/mod.rs
@@ -0,0 +1,17 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+pub mod bro_reader;
diff --git a/brro-compressor/src/preprocessor/mod.rs b/atsc/src/utils/writers/mod.rs
similarity index 100%
rename from brro-compressor/src/preprocessor/mod.rs
rename to atsc/src/utils/writers/mod.rs
diff --git a/atsc/tests/csv/cpu_utilization.csv b/atsc/tests/csv/cpu_utilization.csv
new file mode 100644
index 0000000..1b9dc7a
--- /dev/null
+++ b/atsc/tests/csv/cpu_utilization.csv
@@ -0,0 +1,2855 @@
+time,value
+1730419200,60.486139011651275
+1730419220,65.44663767146203
+1730419240,68.66631313800875
+1730419260,60.99319727891156
+1730419280,56.17527173913043
+1730419300,53.807848084456545
+1730419320,52.31012876127522
+1730419340,35.62985477879095
+1730419360,37.19002740458525
+1730419380,34.84838298
+1730419400,33.60759917051308
+1730419420,44.19748097258705
+1730419440,36.09307359307359
+1730419460,34.67887455724119
+1730419480,35.71475909603022
+1730419500,48.76137344756592
+1730419520,37.707573916565416
+1730419540,34.58996493121122
+1730419560,46.48191956742143
+1730419580,44.41222094827005
+1730419600,46.316139767054906
+1730419620,47.09473966720344
+1730419640,36.95083506606748
+1730419660,35.42657998793647
+1730419680,48.01377118644068
+1730419700,38.17094713955701
+1730419720,34.37643207855974
+1730419740,26.11023405861761
+1730419760,21.30729481
+1730419780,23.96445750935329
+1730419800,37.30611696770295
+1730419820,43.96847848050111
+1730419840,36.69335845501649
+1730419860,37.55643911314779
+1730419880,36.19338557577368
+1730419900,23.81870140227288
+1730419920,35.65270106425973
+1730419940,22.84139214
+1730419960,26.70439234707626
+1730419980,21.02712526081982
+1730420000,21.53653224209623
+1730420020,21.156947057242682
+1730420040,25.81049851436118
+1730420060,22.219263795513548
+1730420080,21.96208846025939
+1730420100,22.53635992931902
+1730420120,19.77565160013197
+1730420140,21.95712077424558
+1730420160,27.08235766374523
+1730420180,25.58124174372523
+1730420200,20.18625217740855
+1730420220,20.94590087385765
+1730420240,21.25181950509461
+1730420260,22.14171762829035
+1730420280,37.76344657934978
+1730420300,45.50954557676795
+1730420320,41.231195566112426
+1730420340,46.377883903948344
+1730420360,34.29653248306098
+1730420380,32.19118142039717
+1730420400,37.315024663378225
+1730420420,25.59994681911853
+1730420440,20.73656270736563
+1730420460,27.43291950340408
+1730420480,20.63555114200596
+1730420500,18.21381142098274
+1730420520,30.620569747224817
+1730420540,21.22963741460851
+1730420560,20.70415846957466
+1730420580,21.64493480441324
+1730420600,23.065456023851468
+1730420620,28.92965553649202
+1730420640,25.725806451612897
+1730420660,21.142473118279568
+1730420680,19.79298831385643
+1730420700,22.04687708083633
+1730420720,26.79917294737544
+1730420740,29.436450839328536
+1730420760,24.18104025
+1730420780,40.60710957262681
+1730420800,20.35392229981694
+1730420820,22.03627150286705
+1730420840,32.17867511
+1730420860,26.096299778389632
+1730420880,19.43680858196447
+1730420900,20.32745758235954
+1730420920,19.039746048541762
+1730420940,24.4175898
+1730420960,34.222784136278946
+1730420980,34.10919731329388
+1730421000,38.467752793860235
+1730421020,38.22957850298197
+1730421040,41.51776615259189
+1730421060,46.236056375659615
+1730421080,48.114347067507715
+1730421100,39.23056443024494
+1730421120,50.03657158055722
+1730421140,52.31752438614195
+1730421160,36.55439721999465
+1730421180,36.15706597568268
+1730421200,39.24971630732261
+1730421220,36.99751694517147
+1730421240,33.29335731894197
+1730421260,43.38666666666667
+1730421280,42.50584697627798
+1730421300,42.986916141278556
+1730421320,31.782282479926288
+1730421340,22.66353572626635
+1730421360,27.19292367
+1730421380,20.786928976325438
+1730421400,34.87138155685405
+1730421420,40.84005376344086
+1730421440,39.432387312186975
+1730421460,36.17666891436278
+1730421480,37.57763975
+1730421500,42.70142491299494
+1730421520,21.41298417894163
+1730421540,22.276476410425598
+1730421560,31.93790220924832
+1730421580,27.70429060632512
+1730421600,22.47681763
+1730421620,22.26557244719494
+1730421640,35.286311459093625
+1730421660,26.052968528876608
+1730421680,30.853682879765792
+1730421700,32.66666666666667
+1730421720,24.129731157936458
+1730421740,24.15926822706484
+1730421760,25.447329476658147
+1730421780,26.376695929768562
+1730421800,29.097347497828558
+1730421820,22.50368583299826
+1730421840,20.786367855722048
+1730421860,29.71688641991835
+1730421880,26.363757317722193
+1730421900,25.14576271186441
+1730421920,35.30390786232608
+1730421940,40.011906336817034
+1730421960,39.39886271324127
+1730421980,39.47792242651993
+1730422000,44.135122504840105
+1730422020,40.99169087347159
+1730422040,32.38830441
+1730422060,24.74517167381974
+1730422080,27.41140735740803
+1730422100,36.91883902176834
+1730422120,25.10621348911312
+1730422140,27.30720606826802
+1730422160,26.944444444444436
+1730422180,25.381358779651908
+1730422200,38.260459475847206
+1730422220,41.15951742627346
+1730422240,32.933647692825865
+1730422260,29.83725136
+1730422280,28.263221966725283
+1730422300,35.587472571314585
+1730422320,34.71244521184752
+1730422340,41.03668798384382
+1730422360,29.36147475287203
+1730422380,26.94425789120215
+1730422400,40.01467938880363
+1730422420,49.44186046511628
+1730422440,53.80097218
+1730422460,49.68632371392723
+1730422480,52.64186548052297
+1730422500,50.88970343218927
+1730422520,38.34000135620804
+1730422540,43.223908537808384
+1730422560,49.81630153762417
+1730422580,27.00262750117901
+1730422600,27.50083194675541
+1730422620,27.219538744132247
+1730422640,31.68099140515691
+1730422660,39.55527933153788
+1730422680,26.05814739688979
+1730422700,26.094219846308047
+1730422720,21.58046555309586
+1730422740,34.84900129524848
+1730422760,31.306502377285213
+1730422780,26.586470230452118
+1730422800,25.58685446
+1730422820,27.18628632650328
+1730422840,36.843167597389495
+1730422860,39.59525756336877
+1730422880,49.02453946026783
+1730422900,49.18165285916347
+1730422920,33.493249263244465
+1730422940,34.78144811478145
+1730422960,26.324659565862206
+1730422980,25.61658445540595
+1730423000,30.28532608695652
+1730423020,21.19064962585259
+1730423040,21.207874541207868
+1730423060,32.79602391026933
+1730423080,35.82693085321472
+1730423100,41.37585238668271
+1730423120,37.07464694014795
+1730423140,37.21776639482375
+1730423160,34.132013201320134
+1730423180,46.734555329260004
+1730423200,46.84457982915039
+1730423220,39.53835709436524
+1730423240,36.65958012224289
+1730423260,36.50531598758502
+1730423280,35.36992840095465
+1730423300,30.10795681727309
+1730423320,44.76462386295731
+1730423340,37.69514237855946
+1730423360,48.81235154394299
+1730423380,46.58610271903323
+1730423400,51.05228325612177
+1730423420,48.15459843258088
+1730423440,52.371202113606344
+1730423460,61.458474004051325
+1730423480,49.65328710494733
+1730423500,49.87977558108469
+1730423520,54.95854063018242
+1730423540,63.021640935751364
+1730423560,53.21765722643266
+1730423580,61.60167037111874
+1730423600,35.57996899642785
+1730423620,37.143428419088245
+1730423640,32.586287158342756
+1730423660,41.311961206896555
+1730423680,48.73438856608562
+1730423700,41.78422412
+1730423720,36.427850655903136
+1730423740,37.19288900725747
+1730423760,42.96404211641613
+1730423780,57.68622280817402
+1730423800,53.098124195842075
+1730423820,52.26154364678126
+1730423840,49.01474530831099
+1730423860,54.05278670953913
+1730423880,45.73539288112827
+1730423900,35.32377919320594
+1730423920,35.006578947368425
+1730423940,46.03695273798367
+1730423960,36.82475613858056
+1730423980,37.13409348536374
+1730424000,39.27639938473885
+1730424020,49.68041445199489
+1730424040,42.20006646726487
+1730424060,40.17384218389244
+1730424080,38.915694947058036
+1730424100,41.75581706338593
+1730424120,52.075165607678784
+1730424140,50.456940831165376
+1730424160,40.57173533753859
+1730424180,35.02558577969297
+1730424200,40.18215912135012
+1730424220,52.95612627073301
+1730424240,58.64218216
+1730424260,53.43331764072903
+1730424280,48.316712402114995
+1730424300,50.83450633420471
+1730424320,54.37479270315091
+1730424340,63.553834857486144
+1730424360,46.21854485776805
+1730424380,42.85230810883985
+1730424400,47.26942970376839
+1730424420,41.207526736158115
+1730424440,35.52765524261961
+1730424460,38.02713848609818
+1730424480,33.76570733684637
+1730424500,31.45962525838501
+1730424520,52.80914069366507
+1730424540,36.13241825
+1730424560,33.04115088658414
+1730424580,35.042111545858475
+1730424600,34.93238242615892
+1730424620,32.870122928915016
+1730424640,42.060397806119084
+1730424660,43.20820039372751
+1730424680,39.036823395264605
+1730424700,35.851851851851855
+1730424720,38.36165873555404
+1730424740,45.68090787716956
+1730424760,39.85813667985268
+1730424780,36.00215299737603
+1730424800,43.21259422320059
+1730424820,34.65829380720155
+1730424840,36.57957244655582
+1730424860,33.11640879208447
+1730424880,31.45511255763493
+1730424900,32.82703245232519
+1730424920,31.44505638604773
+1730424940,21.522111758656152
+1730424960,22.13945372793772
+1730424980,28.923180140892317
+1730425000,34.66495714960524
+1730425020,29.248699545664046
+1730425040,23.60378634212306
+1730425060,18.13502276776876
+1730425080,23.09821127
+1730425100,18.46430234879233
+1730425120,21.980595516895278
+1730425140,19.96267413
+1730425160,30.97084782
+1730425180,25.10418066944482
+1730425200,20.00936266969839
+1730425220,35.34952566776559
+1730425240,29.81247100921079
+1730425260,25.17355260497405
+1730425280,21.169099513430652
+1730425300,34.92754565
+1730425320,37.17080511662904
+1730425340,21.409503470368392
+1730425360,23.25921981094395
+1730425380,20.97596504005827
+1730425400,22.72696727370472
+1730425420,19.84461857879579
+1730425440,31.703260146373918
+1730425460,33.055720801110446
+1730425480,33.32668660684613
+1730425500,34.97022813942597
+1730425520,35.15289699570815
+1730425540,48.34773431831673
+1730425560,37.36874195515209
+1730425580,35.503077334760505
+1730425600,39.11019485269668
+1730425620,52.91345898593045
+1730425640,50.37819420783646
+1730425660,45.80228845130389
+1730425680,32.78732366
+1730425700,33.295401298614365
+1730425720,29.578305264553773
+1730425740,36.18214595713427
+1730425760,40.21753726332751
+1730425780,33.70913942628419
+1730425800,36.16545934601584
+1730425820,31.95348837209302
+1730425840,37.49916683330001
+1730425860,42.32495812
+1730425880,33.68399757
+1730425900,21.11274378840502
+1730425920,23.019422011613162
+1730425940,29.15920100869334
+1730425960,45.94486081
+1730425980,47.07108749747797
+1730426000,33.54696575205287
+1730426020,31.73411953899759
+1730426040,41.02359056377449
+1730426060,22.828567592965502
+1730426080,18.17155006311873
+1730426100,30.88410991636798
+1730426120,35.68500974265941
+1730426140,37.12748065926673
+1730426160,34.73145440042542
+1730426180,34.767971483266216
+1730426200,37.06009745533297
+1730426220,27.660146534919672
+1730426240,32.5992369
+1730426260,22.658834649922362
+1730426280,21.70087976539589
+1730426300,21.788379530916842
+1730426320,22.53679139444334
+1730426340,27.55558523068705
+1730426360,22.13990121479108
+1730426380,34.60972017673049
+1730426400,19.24048949188614
+1730426420,39.57487922705314
+1730426440,37.26411880918319
+1730426460,34.699343779295575
+1730426480,21.359417773920008
+1730426500,25.20244914
+1730426520,27.68452983
+1730426540,29.75707154742097
+1730426560,24.3795329
+1730426580,22.274060250599838
+1730426600,34.10439708659293
+1730426620,29.122807017543863
+1730426640,20.93487814622453
+1730426660,19.174645097391878
+1730426680,31.70421024403325
+1730426700,24.08554867079752
+1730426720,30.84798275745942
+1730426740,30.50656411616496
+1730426760,42.838770813591445
+1730426780,34.74747474747475
+1730426800,37.83599399645245
+1730426820,45.82115333510497
+1730426840,39.386307906421905
+1730426860,39.94904116937106
+1730426880,45.51403130059363
+1730426900,26.68790672451193
+1730426920,20.86910579844142
+1730426940,21.21068650659452
+1730426960,20.01704694
+1730426980,30.84603607801371
+1730427000,23.44431232659532
+1730427020,23.96895038812015
+1730427040,19.38721254818556
+1730427060,34.82012478428249
+1730427080,32.57348530293941
+1730427100,26.91490774
+1730427120,21.863701480319257
+1730427140,22.71247258954083
+1730427160,35.23923763827802
+1730427180,37.49750548792656
+1730427200,19.69717362
+1730427220,20.56539916384631
+1730427240,24.81531229
+1730427260,37.84676354029062
+1730427280,33.83318956937164
+1730427300,32.76707177868797
+1730427320,35.17828632251197
+1730427340,34.98411016949153
+1730427360,44.183845893883074
+1730427380,35.24345809870818
+1730427400,24.30740543420352
+1730427420,30.76513285
+1730427440,51.190875545119084
+1730427460,39.240676147035146
+1730427480,31.91475235692471
+1730427500,33.22814556197126
+1730427520,34.89415023412254
+1730427540,30.22724216754068
+1730427560,18.15109343936382
+1730427580,20.97943974981702
+1730427600,22.99775696002111
+1730427620,29.18175235336713
+1730427640,32.909613969358396
+1730427660,22.64340825226434
+1730427680,21.18661054319503
+1730427700,22.67031062524997
+1730427720,26.69237930575874
+1730427740,34.66869060190074
+1730427760,25.19895673109075
+1730427780,13.85002983491348
+1730427800,19.43430656934307
+1730427820,18.32623538159941
+1730427840,22.059809574896068
+1730427860,25.27326929446837
+1730427880,22.99009900990099
+1730427900,17.997584216883638
+1730427920,21.049823250850398
+1730427940,29.932468220338983
+1730427960,20.52
+1730427980,20.31937964119077
+1730428000,41.424332344213646
+1730428020,23.54715463502916
+1730428040,18.20072454045351
+1730428060,21.881458763917593
+1730428080,21.53169707310671
+1730428100,18.249883013570432
+1730428120,23.32113144758735
+1730428140,29.97060395510422
+1730428160,34.33513657917585
+1730428180,34.91112442580387
+1730428200,35.285406569736125
+1730428220,34.29829242
+1730428240,44.45765622935659
+1730428260,53.20042672356314
+1730428280,40.45776058
+1730428300,38.07283967208709
+1730428320,34.79069767
+1730428340,50.84262208067941
+1730428360,41.31408328320299
+1730428380,31.22733986840338
+1730428400,31.797909871530322
+1730428420,40.900039824771014
+1730428440,44.88556749182625
+1730428460,34.01611820013432
+1730428480,34.40860215053763
+1730428500,36.75643485740753
+1730428520,32.50166333998669
+1730428540,44.98946536739531
+1730428560,46.733119410118114
+1730428580,24.14351227407607
+1730428600,24.68490211853044
+1730428620,22.32809168443497
+1730428640,23.16928354664204
+1730428660,33.22167883700677
+1730428680,29.96464545393903
+1730428700,28.36519182
+1730428720,21.176784167388547
+1730428740,16.30019749835418
+1730428760,20.015765617815152
+1730428780,27.953852274234194
+1730428800,35.46945852227261
+1730428820,36.41063515509601
+1730428840,38.69091395333289
+1730428860,34.19411093970085
+1730428880,28.967475826627897
+1730428900,35.70355860267711
+1730428920,22.36564984034241
+1730428940,24.60502633157789
+1730428960,23.37397034596376
+1730428980,22.41693064635708
+1730429000,29.26583975863225
+1730429020,39.94639866
+1730429040,36.33333333333333
+1730429060,33.03994132159765
+1730429080,38.57947844319224
+1730429100,40.53142702994335
+1730429120,36.56751493990465
+1730429140,43.76506830967051
+1730429160,50.48543689320388
+1730429180,48.98703404
+1730429200,49.71836520827875
+1730429220,52.44043174258367
+1730429240,49.91096748664512
+1730429260,51.367468640756734
+1730429280,46.47172708458509
+1730429300,35.58664078709591
+1730429320,30.059069489613062
+1730429340,33.61670794564563
+1730429360,33.88996075
+1730429380,35.542728237791934
+1730429400,48.30723939515589
+1730429420,36.79771550006641
+1730429440,39.08222563623004
+1730429460,44.96476009758742
+1730429480,35.766666666666666
+1730429500,31.08512193501229
+1730429520,27.06913996627319
+1730429540,23.63337713534823
+1730429560,22.93230540339487
+1730429580,20.32438926712054
+1730429600,31.04180491041805
+1730429620,30.34334475576161
+1730429640,22.11078639744952
+1730429660,17.69195036166965
+1730429680,20.69943912900033
+1730429700,29.47957619777437
+1730429720,24.26090414954262
+1730429740,24.88802727455044
+1730429760,32.60811173784582
+1730429780,38.670654702558984
+1730429800,36.53769445552453
+1730429820,38.686773059237225
+1730429840,44.49394866338609
+1730429860,39.43362361231137
+1730429880,36.670010030090275
+1730429900,39.78393612024425
+1730429920,50.367136336574724
+1730429940,54.624664879356565
+1730429960,48.82478632478632
+1730429980,48.72632702053881
+1730430000,46.63865546218487
+1730430020,37.27703921437638
+1730430040,26.413065394179718
+1730430060,30.469637250317323
+1730430080,33.98622902600441
+1730430100,22.457942030943858
+1730430120,24.41049485220857
+1730430140,23.73957919809448
+1730430160,27.723898067018933
+1730430180,33.299839228295816
+1730430200,27.600905821233518
+1730430220,28.06345009914078
+1730430240,26.582709326072163
+1730430260,22.93351152689609
+1730430280,31.98318430535166
+1730430300,27.673671529915673
+1730430320,26.77456181586193
+1730430340,26.05605471369183
+1730430360,39.60537021969081
+1730430380,41.43482654092287
+1730430400,37.40596453519613
+1730430420,41.39081687132942
+1730430440,55.06904681711015
+1730430460,44.09576626538618
+1730430480,36.537683699609005
+1730430500,46.23255813953488
+1730430520,42.189388851578244
+1730430540,36.65456501277397
+1730430560,34.58524723047825
+1730430580,35.52454153182309
+1730430600,49.191472510065346
+1730430620,51.28568536140919
+1730430640,38.10852095370924
+1730430660,37.74879423952177
+1730430680,37.00243704305443
+1730430700,43.84137792910074
+1730430720,49.41121370266292
+1730430740,38.83952255
+1730430760,34.38598863943738
+1730430780,34.06535603918732
+1730430800,51.61354581673307
+1730430820,26.97154471544715
+1730430840,15.350671926188411
+1730430860,29.97287821657736
+1730430880,23.08156289792909
+1730430900,22.2304883
+1730430920,23.61259338313767
+1730430940,20.12629316135967
+1730430960,19.54524423293013
+1730430980,28.948069241011982
+1730431000,46.901530850992714
+1730431020,37.79516934286871
+1730431040,34.73642278581862
+1730431060,33.676975945017176
+1730431080,41.59095497678175
+1730431100,34.73620212119165
+1730431120,23.7945982
+1730431140,20.90604026845638
+1730431160,21.040299906279287
+1730431180,27.41170268845546
+1730431200,37.530479544838805
+1730431220,23.46870392708752
+1730431240,27.52132124001624
+1730431260,25.26709401709402
+1730431280,32.73733216
+1730431300,23.190921228304408
+1730431320,26.41496825927163
+1730431340,38.582357321779156
+1730431360,27.435587761674718
+1730431380,27.83457249070632
+1730431400,21.46809787319155
+1730431420,20.401292755184492
+1730431440,20.87394518218094
+1730431460,28.349892588614388
+1730431480,21.388741370154012
+1730431500,22.161983471074382
+1730431520,31.26464484166834
+1730431540,35.051338836319715
+1730431560,44.17202306010205
+1730431580,45.07705767548287
+1730431600,37.56539235412475
+1730431620,35.79571971464764
+1730431640,39.99867444326617
+1730431660,49.03717571543193
+1730431680,49.46928486135067
+1730431700,47.58528428093645
+1730431720,43.49206349206349
+1730431740,47.73498304
+1730431760,51.20341356090407
+1730431780,54.26039078761834
+1730431800,21.66666666666667
+1730431820,26.950023353573098
+1730431840,18.52466635681562
+1730431860,32.65210084033613
+1730431880,38.59954479
+1730431900,33.41099720410065
+1730431920,43.33244610061219
+1730431940,45.153789942204206
+1730431960,36.27246415830921
+1730431980,21.46649640432825
+1730432000,22.25023218787316
+1730432020,20.161073825503358
+1730432040,22.83053757759873
+1730432060,18.57543765869304
+1730432080,24.13815876370361
+1730432100,35.24880708929789
+1730432120,32.87774208979703
+1730432140,25.25266046449368
+1730432160,33.69115882117616
+1730432180,22.95028630921395
+1730432200,23.60029788098301
+1730432220,28.96965655218406
+1730432240,39.02143961287721
+1730432260,37.65250033516557
+1730432280,31.318463771941403
+1730432300,32.69749258560259
+1730432320,30.563559605977346
+1730432340,22.689132040008058
+1730432360,16.51907583727279
+1730432380,24.18043710021322
+1730432400,31.11972207375735
+1730432420,21.55712571657112
+1730432440,30.87953479151523
+1730432460,26.18806192117894
+1730432480,23.29117330462863
+1730432500,20.490261920752182
+1730432520,41.72652211854736
+1730432540,35.274842851410995
+1730432560,43.77011802575107
+1730432580,36.90940170940171
+1730432600,34.68679043675705
+1730432620,30.932458665238638
+1730432640,39.73851030110935
+1730432660,38.59073875802998
+1730432680,33.31976665309999
+1730432700,31.121685927940177
+1730432720,24.94380536824012
+1730432740,31.71501477
+1730432760,36.10472195913949
+1730432780,37.22351378279641
+1730432800,37.091503267973856
+1730432820,35.33704883227176
+1730432840,39.81866521235258
+1730432860,46.70972951150585
+1730432880,43.77892730737816
+1730432900,34.98701471665446
+1730432920,39.60668501241694
+1730432940,31.48490043518396
+1730432960,31.48490043518396
+1730432980,41.02271200107512
+1730433000,45.88093806374023
+1730433020,45.88093806374023
+1730433040,35.089610564613935
+1730433060,35.81292948161985
+1730433080,34.42698100851343
+1730433100,32.201309328968904
+1730433120,34.779100262255405
+1730433140,34.76925131904094
+1730433160,45.32049153105281
+1730433180,53.57094821788837
+1730433200,45.45333864
+1730433220,41.32019438444924
+1730433240,45.08928571428571
+1730433260,41.98363338788871
+1730433280,62.30625334045965
+1730433300,58.43305785123967
+1730433320,57.33758307337583
+1730433340,60.8169239
+1730433360,59.14755402359803
+1730433380,60.43747959516814
+1730433400,67.70488480508075
+1730433420,60.63496298272527
+1730433440,59.42711135018827
+1730433460,59.42711135018827
+1730433480,61.44005358338915
+1730433500,58.411494711634404
+1730433520,55.71884559116904
+1730433540,59.04679606487636
+1730433560,57.547611092549275
+1730433580,59.91642345449721
+1730433600,67.12928653237216
+1730433620,33.557449074690446
+1730433640,34.60714763753179
+1730433660,34.60714763753179
+1730433680,32.67214319107727
+1730433700,28.482992744458503
+1730433720,28.482992744458503
+1730433740,31.182147165259348
+1730433760,39.10188379
+1730433780,40.39721678038235
+1730433800,41.64715327490466
+1730433820,40.869623746382175
+1730433840,41.95261166788345
+1730433860,41.14308877
+1730433880,35.11280167890871
+1730433900,43.067966422962364
+1730433920,35.36811440677966
+1730433940,40.581567939549316
+1730433960,49.891642963564955
+1730433980,45.613220069020436
+1730434000,45.919470341845695
+1730434020,53.94454862041253
+1730434040,42.43100649350649
+1730434060,38.77878950187467
+1730434080,47.2276232
+1730434100,32.68588564029169
+1730434120,40
+1730434140,35.94150155007413
+1730434160,43.35788325150027
+1730434180,44.00269541778976
+1730434200,36.109054190508246
+1730434220,34.29368029739777
+1730434240,38.58134987526128
+1730434260,50.932798395185564
+1730434280,46.090007421901355
+1730434300,32.35489440828795
+1730434320,32.923665164550044
+1730434340,37.30455699585728
+1730434360,36.69993346640053
+1730434380,35.66580697705986
+1730434400,33.71920850720151
+1730434420,30.148084202138257
+1730434440,31.691042663611242
+1730434460,34.56404807756159
+1730434480,36.03460801796447
+1730434500,35.29649868447683
+1730434520,44.37909370405026
+1730434540,40.19509587759738
+1730434560,26.01365187713311
+1730434580,22.97784303256807
+1730434600,32.05024855568991
+1730434620,36.45581520279345
+1730434640,35.14989920010405
+1730434660,40.629836484758755
+1730434680,39.97693507903127
+1730434700,47.92586645256256
+1730434720,28.34533351446814
+1730434740,25.19881385631487
+1730434760,28.869901875709232
+1730434780,33.29504763191676
+1730434800,22.12366121295749
+1730434820,27.31948298030723
+1730434840,24.63671510465271
+1730434860,30.10694827470236
+1730434880,29.93750419998656
+1730434900,19.56696070569366
+1730434920,16.36303630363036
+1730434940,29.365184988022357
+1730434960,27.17894458404973
+1730434980,24.61590070446159
+1730435000,20.66599394550959
+1730435020,31.03586188508199
+1730435040,31.88328912466844
+1730435060,25.41496598639456
+1730435080,26.10714766455781
+1730435100,24.67326732673267
+1730435120,27.57547042100988
+1730435140,45.90098482832047
+1730435160,31.053129647154247
+1730435180,21.58757664622767
+1730435200,21.11368909512761
+1730435220,20.585483224167888
+1730435240,27.528275679608438
+1730435260,21.86555638
+1730435280,29.72882936904524
+1730435300,21.55401809473124
+1730435320,28.81627209
+1730435340,37.70788626609442
+1730435360,34.255529877847465
+1730435380,35.05364661143556
+1730435400,37.22627737226277
+1730435420,40.57863999462979
+1730435440,37.66190188577948
+1730435460,35.88813402809841
+1730435480,36.098609525088996
+1730435500,43.95530875761022
+1730435520,49.36181469479532
+1730435540,49.22664881151657
+1730435560,45.90822440087146
+1730435580,51.16648992576882
+1730435600,46.92813113865745
+1730435620,59.59981262129425
+1730435640,62.45946124826261
+1730435660,51.89856343157753
+1730435680,51.449420901118025
+1730435700,47.26509612780937
+1730435720,51.856319528213376
+1730435740,51.59321124304018
+1730435760,73.33952008484688
+1730435780,60.616136255682974
+1730435800,49.69623329283111
+1730435820,46.011576076109364
+1730435840,49.780941949616654
+1730435860,42.17691637046476
+1730435880,37.91560956068901
+1730435900,33.27280065
+1730435920,44.853236831523915
+1730435940,46.12977826146103
+1730435960,36.30760986066452
+1730435980,37.19086021505376
+1730436000,37.363080438142596
+1730436020,35.79670512906023
+1730436040,47.22148009617954
+1730436060,50.08126777732629
+1730436080,46.19241826569499
+1730436100,47.40407791023386
+1730436120,50.67929188966653
+1730436140,44.51406479136115
+1730436160,39.38476036618201
+1730436180,38.62369101047192
+1730436200,34.28629086506121
+1730436220,36.232664602127365
+1730436240,31.58320960993386
+1730436260,45.22458786624841
+1730436280,39.28138061210732
+1730436300,42.51767081790643
+1730436320,34.34208772163712
+1730436340,33.88624001074475
+1730436360,37.9434998
+1730436380,41.10879364865774
+1730436400,42.89370874309578
+1730436420,44.129419613075385
+1730436440,42.618290123874644
+1730436460,31.82182449579271
+1730436480,44.55917394757744
+1730436500,29.52880285003697
+1730436520,45.54886211512718
+1730436540,38.87951400607492
+1730436560,36.72069406147017
+1730436580,32.28084470700075
+1730436600,33.78178393649939
+1730436620,48.87043189368771
+1730436640,48.77589454
+1730436660,40.69610879224451
+1730436680,35.39546872880206
+1730436700,34.339747843397475
+1730436720,42.27260531258385
+1730436740,36.18319586937571
+1730436760,32.66088359895098
+1730436780,33.47785485592316
+1730436800,36.52971092077088
+1730436820,41.89736664415935
+1730436840,41.89736664415935
+1730436860,49.08722109533469
+1730436880,36.29191321499014
+1730436900,38.32791884903534
+1730436920,38.32791884903534
+1730436940,45.27709071461996
+1730436960,46.22025832766825
+1730436980,38.90647094471753
+1730437000,38.47301328993762
+1730437020,38.75397415950754
+1730437040,40.073279956574844
+1730437060,45.59799539482595
+1730437080,38.03548539431964
+1730437100,38.14989871708305
+1730437120,41.73942243116185
+1730437140,42.316623412102174
+1730437160,39.54475048824837
+1730437180,53.430407975862515
+1730437200,52.61909602222674
+1730437220,54.58445040214477
+1730437240,64.06462123269074
+1730437260,64.06462123269074
+1730437280,53.687971952535065
+1730437300,51.807802093244526
+1730437320,51.807802093244526
+1730437340,51.807802093244526
+1730437360,51.807802093244526
+1730437380,51.807802093244526
+1730437400,51.807802093244526
+1730437420,51.807802093244526
+1730437440,49.52978056426332
+1730437460,56.59004352557127
+1730437480,51.254264499297605
+1730437500,48.10083806434171
+1730437520,51.04173645072687
+1730437540,49.97971602434077
+1730437560,56.163734776725306
+1730437580,57.738896366083445
+1730437600,58.66365441226809
+1730437620,39.18223666395884
+1730437640,35.34442440034743
+1730437660,42.08784389436668
+1730437680,37.268722466960355
+1730437700,41.18979415095597
+1730437720,49.40983830251757
+1730437740,51.09665302837212
+1730437760,39.22090326101787
+1730437780,35.84058759521219
+1730437800,36.421832884097036
+1730437820,44.18899953217938
+1730437840,44.08376253727297
+1730437860,44.08376253727297
+1730437880,44.08376253727297
+1730437900,44.08376253727297
+1730437920,44.08376253727297
+1730437940,44.08376253727297
+1730437960,44.558507783145465
+1730437980,45.25078584119175
+1730438000,44.254858148825235
+1730438020,51.3671875
+1730438040,44.62227045
+1730438060,40.969044414535674
+1730438080,41.304048471495456
+1730438100,40.13157894736842
+1730438120,53.83052225157488
+1730438140,40.16890281277668
+1730438160,37.31045490822027
+1730438180,34.58672086720867
+1730438200,58.986673882161945
+1730438220,66.96249833177632
+1730438240,61.98671096345515
+1730438260,64.63950266909926
+1730438280,65.49518615767857
+1730438300,79.50915465523958
+1730438320,74.15551085524034
+1730438340,61.548378341701714
+1730438360,53.856989535819686
+1730438380,62.40905416329831
+1730438400,58.72414022483869
+1730438420,55.830243098475485
+1730438440,52.474029524330234
+1730438460,52.15139442231076
+1730438480,51.72295442
+1730438500,48.50563782094824
+1730438520,50.77615751629595
+1730438540,63.10699173886762
+1730438560,57.09787633147987
+1730438580,58.347849390325614
+1730438600,58.347849390325614
+1730438620,58.347849390325614
+1730438640,50.26401978477375
+1730438660,51.13435674500747
+1730438680,53.36010709504685
+1730438700,59.18435233850043
+1730438720,51.35725003410177
+1730438740,48.771013327975346
+1730438760,56.349259086982514
+1730438780,54.25416610215418
+1730438800,44.59104678559408
+1730438820,54.02556331532481
+1730438840,57.82784129119032
+1730438860,58.96522140830199
+1730438880,50.74385728710872
+1730438900,49.284898945813474
+1730438920,47.003367
+1730438940,49.03405696076479
+1730438960,60.076252723311555
+1730438980,57.54885723749587
+1730439000,50.57633025063664
+1730439020,50.06292640922037
+1730439040,46.546326517018585
+1730439060,49.10665575559193
+1730439080,57.273144806771214
+1730439100,60.806255898611305
+1730439120,53.83994076866124
+1730439140,44.64164747324211
+1730439160,46.507654905052895
+1730439180,46.38795986622074
+1730439200,46.96031043638097
+1730439220,52.05990016638935
+1730439240,58.092061374249504
+1730439260,48.29733938121996
+1730439280,50.39018208497299
+1730439300,48.04585372393913
+1730439320,53.669724770642205
+1730439340,56.69660812441346
+1730439360,54.399679294447786
+1730439380,53.80830412510112
+1730439400,53.80830412510112
+1730439420,53.80830412510112
+1730439440,53.80830412510112
+1730439460,46.89854682454252
+1730439480,41.68616701210136
+1730439500,39.62634535977797
+1730439520,36.40274230407313
+1730439540,35.70464362850972
+1730439560,35.50093758371283
+1730439580,47.88928713659896
+1730439600,47.13050582068214
+1730439620,37.40411377367456
+1730439640,43.45424292845258
+1730439660,39.35300871209563
+1730439680,35.43980079
+1730439700,45.00235579188261
+1730439720,44.42262106332543
+1730439740,50.08814755
+1730439760,48.08102345415778
+1730439780,49.88662131519274
+1730439800,54.26648721399731
+1730439820,46.71300893743793
+1730439840,54.43674412284278
+1730439860,41.98303715670436
+1730439880,37.63788001076137
+1730439900,40.42738478027867
+1730439920,39.89651416122004
+1730439940,45.88976800321845
+1730439960,46.354976015134106
+1730439980,41.98835886800027
+1730440000,52.548625083836356
+1730440020,39.53504043126685
+1730440040,39.17806355038443
+1730440060,42.34547276979619
+1730440080,49.05357501
+1730440100,49.52192323
+1730440120,48.31483081832081
+1730440140,59.736397431564725
+1730440160,49.52916469073911
+1730440180,45.15845665247652
+1730440200,37.93890567304465
+1730440220,41.06365614798694
+1730440240,44.62981574539363
+1730440260,52.2682976
+1730440280,38.31756618798067
+1730440300,39.64472878241339
+1730440320,47.48674585598282
+1730440340,55.30825425001707
+1730440360,42.91596355045562
+1730440380,36.13371304583672
+1730440400,39.73980686695279
+1730440420,34.43636610722294
+1730440440,44.582521663196076
+1730440460,64.81700895208003
+1730440480,57.595149882115194
+1730440500,62.34407128169978
+1730440520,56.451502965033065
+1730440540,54.55348647546554
+1730440560,63.33223358627515
+1730440580,66.24126813541108
+1730440600,67.91932773109244
+1730440620,63.890537421694695
+1730440640,59.87578487578487
+1730440660,52.167756725777096
+1730440680,59.85445724681625
+1730440700,41.75672884972809
+1730440720,35.11024468943264
+1730440740,37.678180733275866
+1730440760,40.48305201042953
+1730440780,50.41032428855063
+1730440800,61.91760501651945
+1730440820,66.31117111036112
+1730440840,53.39262852907486
+1730440860,58.89763779527559
+1730440880,56.19967793880837
+1730440900,56.524941412788756
+1730440920,63.80490398033092
+1730440940,52.663170967296814
+1730440960,51.07851018220793
+1730440980,51.76919983916365
+1730441000,47.57967729030537
+1730441020,61.64134964376932
+1730441040,65.09258025842547
+1730441060,66.61782336485946
+1730441080,63.78699031774172
+1730441100,67.38246082027342
+1730441120,72.85903375
+1730441140,75.47132757
+1730441160,78.25263727346497
+1730441180,67.71917808219177
+1730441200,65.24618678084025
+1730441220,66.93850267379679
+1730441240,70.32138606004497
+1730441260,61.59459459459459
+1730441280,65.10224606101241
+1730441300,65.96147347076715
+1730441320,62.53968253968254
+1730441340,70.85075797
+1730441360,75.82476651212792
+1730441380,75.82476651212792
+1730441400,75.82476651212792
+1730441420,75.82476651212792
+1730441440,61.01271098145447
+1730441460,66.4716169
+1730441480,71.06872739230516
+1730441500,74.37487224909724
+1730441520,67.92328042328042
+1730441540,74.95237469618341
+1730441560,72.18300298669563
+1730441580,61.67797407
+1730441600,57.03172978505629
+1730441620,47.64321004139404
+1730441640,52.410732297760745
+1730441660,55.00470240494424
+1730441680,45.57073302677847
+1730441700,50.51017005668555
+1730441720,49.37316617764737
+1730441740,50.675030683212874
+1730441760,55.82980412916887
+1730441780,62.91591046581972
+1730441800,65.16554526680434
+1730441820,63.45539280958722
+1730441840,64.66978954
+1730441860,65.72245676154161
+1730441880,49.08534512070724
+1730441900,57.38261738261738
+1730441920,43.85296119809394
+1730441940,37.76459484966968
+1730441960,36.34749441010128
+1730441980,38.388144539179855
+1730442000,37.12136613132751
+1730442020,35.378502155172406
+1730442040,35.00366007852532
+1730442060,37.402201299562385
+1730442080,46.02073794775114
+1730442100,38.523047977422394
+1730442120,35.73678214836159
+1730442140,34.52746958962706
+1730442160,40.01340482573727
+1730442180,40.01340482573727
+1730442200,48.332097850259444
+1730442220,32.49780420241876
+1730442240,41.683353357362165
+1730442260,38.63406066337904
+1730442280,31.79823080559119
+1730442300,32.64110756123536
+1730442320,32.84710605553697
+1730442340,33.22717622080679
+1730442360,48.25025092004015
+1730442380,52.49618776105549
+1730442400,52.520390426527605
+1730442420,51.84321045263649
+1730442440,47.69106130164731
+1730442460,30.581325502235867
+1730442480,46.04050326313665
+1730442500,40.12942824738141
+1730442520,33.27097808658471
+1730442540,38.26409102937285
+1730442560,30.168050212593638
+1730442580,34.95178140905438
+1730442600,29.530156165858912
+1730442620,34.47103526431571
+1730442640,42.25286746668435
+1730442660,34.89959839
+1730442680,42.64420827484794
+1730442700,37.5369723
+1730442720,34.202350398487106
+1730442740,22.01544248663631
+1730442760,21.64119601328904
+1730442780,37.78460306
+1730442800,24.66724633803759
+1730442820,16.08214428590479
+1730442840,19.19456066945607
+1730442860,20.34626225
+1730442880,15.68251026082351
+1730442900,15.28283162421436
+1730442920,22.38408833898756
+1730442940,17.43725231175694
+1730442960,24.181327007972587
+1730442980,31.552811703127098
+1730443000,34.41523605150215
+1730443020,42.82174903
+1730443040,46.92859992022337
+1730443060,30.943143812709028
+1730443080,34.20895125357452
+1730443100,30.98106712564544
+1730443120,26.41030776420322
+1730443140,22.54758418740849
+1730443160,19.83668591913961
+1730443180,24.16874753581285
+1730443200,18.544679149730133
+1730443220,21.00946792905721
+1730443240,16.502068597357532
+1730443260,29.654945777193557
+1730443280,23.75831709120237
+1730443300,22.620736237757512
+1730443320,25.14499033397773
+1730443340,19.59436695640455
+1730443360,21.48571428571429
+1730443380,25.715437323659813
+1730443400,33.12583668005355
+1730443420,34.213354830041645
+1730443440,30.53005428
+1730443460,34.10346894
+1730443480,31.09371891
+1730443500,30.03899422
+1730443520,26.81043663471778
+1730443540,33.1
+1730443560,38.53583799626965
+1730443580,47.30764066066873
+1730443600,37.18378342860898
+1730443620,32.62343354105321
+1730443640,21.64294867471505
+1730443660,24.68857672939306
+1730443680,32.15514958962139
+1730443700,21.73032797948441
+1730443720,17.54432742301027
+1730443740,19.937487530757462
+1730443760,18.685558438102838
+1730443780,23.89202256244964
+1730443800,16.96505823627288
+1730443820,30.68592057761733
+1730443840,28.82632831086439
+1730443860,21.27297481279782
+1730443880,23.72609335294904
+1730443900,25.92442645074224
+1730443920,19.42607775720709
+1730443940,36.11037934668072
+1730443960,37.563762497449495
+1730443980,28.06356046323727
+1730444000,36.30980105130082
+1730444020,21.41451164053485
+1730444040,32.94615281321337
+1730444060,34.68578603052931
+1730444080,23.63270777479893
+1730444100,24.82498653742596
+1730444120,25.18718381112985
+1730444140,23.94691667767067
+1730444160,33.492533297457285
+1730444180,27.083615666079407
+1730444200,24.406104844061048
+1730444220,28.42714785887423
+1730444240,40.49067583652956
+1730444260,37.081291535208436
+1730444280,36.97541259683395
+1730444300,37.26527015117619
+1730444320,42.41773002014775
+1730444340,37.55874760045012
+1730444360,44.679128146298666
+1730444380,33.53389490573606
+1730444400,33.63974315647178
+1730444420,36.69486445
+1730444440,34.658900716341975
+1730444460,43.59774716699464
+1730444480,46.41222401723209
+1730444500,42.46117554550816
+1730444520,36.76260461318636
+1730444540,36.86672904691886
+1730444560,36.73578954432588
+1730444580,40.95726951536564
+1730444600,40.95726951536564
+1730444620,50.124302895921524
+1730444640,44.02350081037277
+1730444660,44.02350081037277
+1730444680,35.30753236300221
+1730444700,34.48160085952189
+1730444720,35.86591428187559
+1730444740,32.56941676630796
+1730444760,46.55104454079622
+1730444780,43.50271002710027
+1730444800,42.11298606016141
+1730444820,39.53550453817405
+1730444840,27.62071049627292
+1730444860,27.626847782660814
+1730444880,41.736060222503504
+1730444900,27.42011436
+1730444920,24.03318614
+1730444940,26.37039020145907
+1730444960,19.51170835550825
+1730444980,32.26495726495726
+1730445000,45.62019613040021
+1730445020,45.82550336
+1730445040,46.435039238044126
+1730445060,40.29132700788454
+1730445080,43.48115595
+1730445100,46.15801599132674
+1730445120,36.08828207847296
+1730445140,31.04425256341069
+1730445160,19.106234947819107
+1730445180,32.46021220159151
+1730445200,37.30815428
+1730445220,21.8247347
+1730445240,19.88784298017224
+1730445260,20.71327825911747
+1730445280,19.96553095585311
+1730445300,27.48933377092222
+1730445320,27.48933377092222
+1730445340,23.158888589155648
+1730445360,19.02820698270761
+1730445380,20.93833780160858
+1730445400,22.31525784157363
+1730445420,30.684840425531913
+1730445440,31.03610253657227
+1730445460,25.57229115519873
+1730445480,21.667907669396868
+1730445500,24.299873611388282
+1730445520,32.16358663019548
+1730445540,34.69182895001004
+1730445560,36.752941176470586
+1730445580,36.34086909435495
+1730445600,44.12963581690611
+1730445620,33.37951052180223
+1730445640,38.08656957928803
+1730445660,21.52108934
+1730445680,20.79187817258883
+1730445700,22.640241854215652
+1730445720,23.69263607257204
+1730445740,27.900478978179883
+1730445760,32.11033566
+1730445780,33.100171571862205
+1730445800,41.07622114868492
+1730445820,22.10019659684089
+1730445840,37.78715424285045
+1730445860,36.24941522421974
+1730445880,38.941728064300065
+1730445900,36.76421571895207
+1730445920,37.137330754352035
+1730445940,48.64082895976315
+1730445960,42.86954199492589
+1730445980,40.35455714189052
+1730446000,21.53087735660516
+1730446020,31.65064102564103
+1730446040,35.50180892402519
+1730446060,41.88798493915148
+1730446080,31.85888012829079
+1730446100,28.86281829089455
+1730446120,22.70759805553706
+1730446140,28.24302134646962
+1730446160,31.116879659211932
+1730446180,35.25405621953662
+1730446200,22.74989909861429
+1730446220,24.17838811
+1730446240,20.956153897804338
+1730446260,26.11481975967957
+1730446280,23.63806719314683
+1730446300,29.450474898236088
+1730446320,22.20369505769359
+1730446340,29.61087420042644
+1730446360,44.80021346141018
+1730446380,29.75741239892183
+1730446400,38.96624195455143
+1730446420,36.12269608502623
+1730446440,45.93449458605152
+1730446460,53.694288544073956
+1730446480,60.55862906490353
+1730446500,48.44778302200359
+1730446520,48.44778302200359
+1730446540,47.71081134959133
+1730446560,46.96767298958055
+1730446580,50.46380933035412
+1730446600,43.58356750970159
+1730446620,40.521216171065824
+1730446640,40.58613967869866
+1730446660,48.62531545
+1730446680,39.49875025332703
+1730446700,40.03760660801827
+1730446720,37.31465664502904
+1730446740,39.10178836
+1730446760,43.27321309285237
+1730446780,49.01735665563585
+1730446800,37.69933215210577
+1730446820,35.48408755203438
+1730446840,33.73799359658485
+1730446860,44.42885771543086
+1730446880,41.756558796111335
+1730446900,35.614487249050455
+1730446920,40.735068912710574
+1730446940,40.873475505893566
+1730446960,33.54145686
+1730446980,45.12333712146534
+1730447000,50.35152326749247
+1730447020,40.92591346477357
+1730447040,34.74265929822225
+1730447060,38.27242968802191
+1730447080,34.86592588
+1730447100,34.24593031077627
+1730447120,35.58582740571735
+1730447140,40.212364015373595
+1730447160,50.04080522306855
+1730447180,44.33943427620632
+1730447200,50.74138392
+1730447220,46.56885628542848
+1730447240,52.78113663845224
+1730447260,61.39823717948718
+1730447280,49.63430181842582
+1730447300,51.47440591093656
+1730447320,59.19329491880566
+1730447340,54.388006571741506
+1730447360,47.85757392878696
+1730447380,50.75247524752476
+1730447400,50.75247524752476
+1730447420,48.72585285655569
+1730447440,51.160477453580896
+1730447460,54.49190982330242
+1730447480,42.93542849597253
+1730447500,46.17875211
+1730447520,48.47961758066658
+1730447540,61.065301236866596
+1730447560,62.36602903
+1730447580,34.34479054779807
+1730447600,37.05664373
+1730447620,49.913113220157726
+1730447640,39.525850522368074
+1730447660,33.547394474546785
+1730447680,33.547394474546785
+1730447700,33.547394474546785
+1730447720,33.547394474546785
+1730447740,33.547394474546785
+1730447760,33.547394474546785
+1730447780,37.13522537562604
+1730447800,40.76119905692152
+1730447820,37.64059989287627
+1730447840,45.04137746930059
+1730447860,48.27310840487123
+1730447880,47.15893862235326
+1730447900,46.54344537262382
+1730447920,47.30805559303738
+1730447940,50.17761989342806
+1730447960,41.154799383749754
+1730447980,41.18358299
+1730448000,47.748830995323985
+1730448020,46.61136013915836
+1730448040,51.371588047103664
+1730448060,39.97438835344072
+1730448080,37.24435590969456
+1730448100,35.52126109418466
+1730448120,36.50857449
+1730448140,41.46140085317187
+1730448160,36.04321404456448
+1730448180,35.15477792732167
+1730448200,30.060862662079916
+1730448220,30.060862662079916
+1730448240,45.56038808966209
+1730448260,45.25203580321691
+1730448280,37.84554778238131
+1730448300,37.21323678689856
+1730448320,35.44177375324589
+1730448340,38.18365287588295
+1730448360,41.07466291004226
+1730448380,51.06761565836299
+1730448400,49.31409343859195
+1730448420,50.37751677852349
+1730448440,35.39974056120707
+1730448460,34.317691477885646
+1730448480,41.96368071554411
+1730448500,34.06145624582498
+1730448520,36.734141222666025
+1730448540,39.29097976142608
+1730448560,36.12494032599059
+1730448580,37.56123750083887
+1730448600,27.06761853448276
+1730448620,27.06761853448276
+1730448640,27.06761853448276
+1730448660,27.06761853448276
+1730448680,21.83423640377283
+1730448700,26.22917650232684
+1730448720,26.22917650232684
+1730448740,26.22917650232684
+1730448760,26.22917650232684
+1730448780,26.22917650232684
+1730448800,28.08988764044944
+1730448820,25.17280719414804
+1730448840,29.29394426580922
+1730448860,29.29394426580922
+1730448880,29.29394426580922
+1730448900,37.94034282629235
+1730448920,40.82730230396388
+1730448940,50.739506995336434
+1730448960,50.739506995336434
+1730448980,41.848558340070056
+1730449000,37.70755222281735
+1730449020,37.70755222281735
+1730449040,36.67318454588362
+1730449060,41.06218487394958
+1730449080,32.75277853076715
+1730449100,23.62763656030287
+1730449120,25.15811197656614
+1730449140,25.079322216971576
+1730449160,31.760058914105908
+1730449180,30.24331389503318
+1730449200,28.145074946466806
+1730449220,23.16496972116856
+1730449240,37.03877633167852
+1730449260,29.91667231217397
+1730449280,20.09816927566994
+1730449300,23.00074109007613
+1730449320,21.71229200214707
+1730449340,20.38245961094626
+1730449360,28.335543766578247
+1730449380,26.51993021070997
+1730449400,44.46682782702122
+1730449420,27.698053862787496
+1730449440,39.46630174478276
+1730449460,30.25589225589226
+1730449480,30.25589225589226
+1730449500,30.25589225589226
+1730449520,30.25589225589226
+1730449540,22.79179280188362
+1730449560,28.73057167633914
+1730449580,23.477794895840308
+1730449600,24.90047770700637
+1730449620,25.14662757
+1730449640,32.222295996281794
+1730449660,39.068581398469995
+1730449680,35.86261980830671
+1730449700,36.53324512074099
+1730449720,32.882547839610524
+1730449740,37.437994369218394
+1730449760,43.20592355413248
+1730449780,54.086991650956094
+1730449800,51.51696606786427
+1730449820,51.51048295264251
+1730449840,54.25772921108742
+1730449860,53.499565478975875
+1730449880,62.24344645397277
+1730449900,44.12465128937878
+1730449920,33.990246509452874
+1730449940,34.27255124524271
+1730449960,34.80278422273782
+1730449980,31.21281773931855
+1730450000,43.50019917673616
+1730450020,41.73836461
+1730450040,35.39929281
+1730450060,35.45784591
+1730450080,36.53846153846154
+1730450100,41.84114324
+1730450120,48.03478729859098
+1730450140,45.30416221985059
+1730450160,34.44762602383965
+1730450180,48.86685552407932
+1730450200,55.75308968343137
+1730450220,49.67188964778358
+1730450240,45.71600965406275
+1730450260,41.10828196
+1730450280,29.64016404286281
+1730450300,18.85950303111052
+1730450320,33.79837681937085
+1730450340,23.17180022915684
+1730450360,24.37945603379984
+1730450380,25.15188335358445
+1730450400,32.03040904198062
+1730450420,29.08440439909898
+1730450440,34.53328873
+1730450460,29.54484244546189
+1730450480,22.914072229140718
+1730450500,23.79799609979154
+1730450520,34.99865519096288
+1730450540,25.226908702616118
+1730450560,28.30554997656825
+1730450580,21.81636726546906
+1730450600,23.64721485411141
+1730450620,18.57865318072931
+1730450640,26.391683433936958
+1730450660,20.440463362068968
+1730450680,36.27714716882094
+1730450700,51.304522344126305
+1730450720,37.00536886060847
+1730450740,38.71788009
+1730450760,34.907625125965744
+1730450780,37.36131533217484
+1730450800,40.61413293091005
+1730450820,40.61413293091005
+1730450840,48.82452004041765
+1730450860,30.761000401230444
+1730450880,36.03982892274793
+1730450900,30.41024960169942
+1730450920,35.97184407995219
+1730450940,35.97184407995219
+1730450960,35.97184407995219
+1730450980,23.92756915675531
+1730451000,24.50066577896138
+1730451020,20.61654439
+1730451040,39.50192078420983
+1730451060,36.67909950712668
+1730451080,26.947030067635442
+1730451100,19.73001728953318
+1730451120,22.29939312
+1730451140,21.84845850331037
+1730451160,23.052035589107582
+1730451180,30.15218981
+1730451200,24.53214262262788
+1730451220,20.23156203988757
+1730451240,20.20370124
+1730451260,16.834791903266748
+1730451280,21.20909151503033
+1730451300,26.71735360810991
+1730451320,28.82678583338891
+1730451340,21.99747994
+1730451360,23.97219463753724
+1730451380,24.52326976930257
+1730451400,25.385433280170123
+1730451420,25.61033021723048
+1730451440,31.364855143548493
+1730451460,39.799172264061326
+1730451480,36.87203157401833
+1730451500,36.87203157401833
+1730451520,34.09618573797678
+1730451540,34.55490315178511
+1730451560,33.74734325185972
+1730451580,23.32827249323388
+1730451600,32.22018598
+1730451620,23.272849462365592
+1730451640,36.64471031052561
+1730451660,20.74148972025615
+1730451680,26.15291262135922
+1730451700,34.33246161067525
+1730451720,24.98662744049211
+1730451740,26.15996762881036
+1730451760,28.35699246077566
+1730451780,28.58
+1730451800,33.60879706764412
+1730451820,26.4063921
+1730451840,23.83155317
+1730451860,17.12754697009791
+1730451880,20.84893048128342
+1730451900,20.84893048128342
+1730451920,21.07836570663094
+1730451940,22.86689419795222
+1730451960,33.19057815845824
+1730451980,25.73719255333605
+1730452000,26.84594953519256
+1730452020,37.62641484160472
+1730452040,27.00845665961945
+1730452060,31.21622523183668
+1730452080,37.51762099751628
+1730452100,38.907096515283264
+1730452120,45.598344790762866
+1730452140,46.69629826190792
+1730452160,46.49321266968326
+1730452180,46.49321266968326
+1730452200,46.49321266968326
+1730452220,46.49321266968326
+1730452240,46.49321266968326
+1730452260,43.492829379439755
+1730452280,41.183542612877375
+1730452300,37.50164279
+1730452320,37.50164279
+1730452340,47.99196787148594
+1730452360,33.43110459302713
+1730452380,34.34837175506969
+1730452400,36.33144002672903
+1730452420,47.604750467040304
+1730452440,37.50666311
+1730452460,34.27011188066063
+1730452480,41.54155495978552
+1730452500,33.852191844608534
+1730452520,49.1098021
+1730452540,53.95345717926363
+1730452560,48.397671773600045
+1730452580,47.37299465240642
+1730452600,44.12955465587045
+1730452620,57.13812301
+1730452640,67.62585169015016
+1730452660,59.44796868673235
+1730452680,40.73138298
+1730452700,41.66441136671177
+1730452720,37.273274881199384
+1730452740,33.20734699290498
+1730452760,48.719839142091146
+1730452780,44.573485465905996
+1730452800,37.145348447246626
+1730452820,37.145348447246626
+1730452840,39.09603004291845
+1730452860,33.76926669780476
+1730452880,34.34640304588872
+1730452900,31.36684944176521
+1730452920,43.69592497193052
+1730452940,33.23321318916033
+1730452960,35.0016818
+1730452980,37.15608643
+1730453000,35.78169339404738
+1730453020,36.05909988425138
+1730453040,27.745435300344013
+1730453060,33.91350561349897
+1730453080,25.56527590847914
+1730453100,22.06851119894598
+1730453120,32.46883795737837
+1730453140,23.419078242229368
+1730453160,22.50731188513693
+1730453180,29.44058258854684
+1730453200,23.84293617595466
+1730453220,19.11337209302326
+1730453240,20.55042185616714
+1730453260,24.87432133521013
+1730453280,29.035052645608424
+1730453300,29.728461280590007
+1730453320,25.20766345123258
+1730453340,21.61892736093573
+1730453360,24.079113293953668
+1730453380,21.14620513
+1730453400,31.85628743
+1730453420,37.537913754450756
+1730453440,42.11126770015435
+1730453460,33.67374182352156
+1730453480,33.67374182352156
+1730453500,37.66741811430845
+1730453520,23.41130340724716
+1730453540,19.027694759912333
+1730453560,28.993101618466437
+1730453580,27.75838926174497
+1730453600,23.74900186318871
+1730453620,21.69978225367447
+1730453640,24.94662396583934
+1730453660,24.94662396583934
+1730453680,36.98280494357872
+1730453700,31.81575789894681
+1730453720,28.52637229902027
+1730453740,27.707290309306632
+1730453760,24.10086533261222
+1730453780,25.65226184858087
+1730453800,22.52739997359039
+1730453820,20.03610108303249
+1730453840,20.03610108303249
+1730453860,26.08551321205683
+1730453880,28.000000000000004
+1730453900,26.997482443354983
+1730453920,23.66600133067199
+1730453940,22.11399237
+1730453960,23.74423115510668
+1730453980,22.41038482
+1730454000,27.84608227553665
+1730454020,38.91718038059501
+1730454040,36.082952362022134
+1730454060,37.05246187949217
+1730454080,48.16358943815763
+1730454100,30.276770119575442
+1730454120,23.759599332220372
+1730454140,24.93812295136798
+1730454160,20.37892481756712
+1730454180,22.14729251518996
+1730454200,17.465142404017712
+1730454220,21.249748305255387
+1730454240,21.05546943
+1730454260,28.622445605449382
+1730454280,28.622445605449382
+1730454300,28.622445605449382
+1730454320,28.622445605449382
+1730454340,30.01867414
+1730454360,25.29131355932203
+1730454380,33.00053248136315
+1730454400,33.00053248136315
+1730454420,33.00053248136315
+1730454440,34.0544172
+1730454460,35.32886440565474
+1730454480,46.74579887970125
+1730454500,46.74579887970125
+1730454520,37.63644992880873
+1730454540,45.41622055674518
+1730454560,47.09146830703397
+1730454580,47.06632653061224
+1730454600,44.71411749313097
+1730454620,48.04454101032048
+1730454640,54.91059514278089
+1730454660,51.362314921235885
+1730454680,51.35135135135135
+1730454700,55.179363160016116
+1730454720,54.49288847534228
+1730454740,60.16622744780054
+1730454760,49.08198264846325
+1730454780,49.80229207157697
+1730454800,55.78926546221812
+1730454820,39.156297005332966
+1730454840,46.15692287180855
+1730454860,42.26474193109083
+1730454880,33.68818218568926
+1730454900,28.69138788123069
+1730454920,23.47935917037825
+1730454940,22.73478289604124
+1730454960,35.49820883640706
+1730454980,26.239710901425422
+1730455000,23.912749864937872
+1730455020,25.55398396982555
+1730455040,22.43662252977577
+1730455060,31.48160468365378
+1730455080,38.16589800147049
+1730455100,29.514706872181463
+1730455120,19.11676145868183
+1730455140,22.63074064242102
+1730455160,28.36959405677899
+1730455180,41.92794547224927
+1730455200,45.95495679390352
+1730455220,51.14988634844231
+1730455240,49.68612261252838
+1730455260,52.24127656756936
+1730455280,46.630136986301366
+1730455300,40.79239943
+1730455320,40.15100444923824
+1730455340,49.00419997
+1730455360,41.908319185059426
+1730455380,37.429537767756486
+1730455400,35.46679176998861
+1730455420,35.28391272436111
+1730455440,19.677721315859863
+1730455460,28.737630382455198
+1730455480,23.38656460340504
+1730455500,23.78913567032379
+1730455520,23.78913567032379
+1730455540,23.32930757
+1730455560,23.38806567881458
+1730455580,28.72103236
+1730455600,28.72103236
+1730455620,35.47789250728091
+1730455640,37.44640943193998
+1730455660,37.44640943193998
+1730455680,26.11341447104454
+1730455700,26.34963161
+1730455720,26.93336901257693
+1730455740,30.25520483546004
+1730455760,38.74017845252364
+1730455780,26.670733360444633
+1730455800,24.46628482166103
+1730455820,20.51647978253483
+1730455840,20.51647978253483
+1730455860,24.41852625901706
+1730455880,25.232022434399408
+1730455900,30.17258612647431
+1730455920,37.34061930783242
+1730455940,37.16908341093521
+1730455960,40.648562087142196
+1730455980,40.648562087142196
+1730456000,40.32301480484522
+1730456020,41.13197352424693
+1730456040,41.13197352424693
+1730456060,41.43208890663472
+1730456080,53.229368850813195
+1730456100,40.86282440175261
+1730456120,37.82038345105954
+1730456140,33.68470525043642
+1730456160,35.68561872909699
+1730456180,31.50345197399289
+1730456200,23.40552750465054
+1730456220,24.82827609203068
+1730456240,22.20369505769359
+1730456260,20.09752240379547
+1730456280,22.33591383372602
+1730456300,22.156563309209222
+1730456320,20.70950469
+1730456340,28.069474945098822
+1730456360,33.61128059114636
+1730456380,33.61128059114636
+1730456400,33.61128059114636
+1730456420,33.61128059114636
+1730456440,33.61128059114636
+1730456460,33.61128059114636
+1730456480,33.61128059114636
+1730456500,33.61128059114636
+1730456520,33.61128059114636
+1730456540,33.61128059114636
+1730456560,33.61128059114636
+1730456580,33.61128059114636
+1730456600,24.77495633481123
+1730456620,23.12360914424439
+1730456640,23.58610534770096
+1730456660,27.72038105460888
+1730456680,21.34983431392439
+1730456700,40.52843117875733
+1730456720,40.52843117875733
+1730456740,40.52843117875733
+1730456760,40.52843117875733
+1730456780,39.60422878828951
+1730456800,33.06419324703056
+1730456820,33.06419324703056
+1730456840,33.06419324703056
+1730456860,33.06419324703056
+1730456880,33.06419324703056
+1730456900,33.06419324703056
+1730456920,33.06419324703056
+1730456940,36.37520938023451
+1730456960,39.80634749865519
+1730456980,26.793163773381778
+1730457000,23.98231239432956
+1730457020,35.91699306
+1730457040,34.21494261
+1730457060,28.714285714285708
+1730457080,26.731811793052668
+1730457100,23.59369761984579
+1730457120,25.014989008060763
+1730457140,20.14671557185729
+1730457160,27.44639637
+1730457180,26.28563815614175
+1730457200,29.56265487911058
+1730457220,34.80310777608075
+1730457240,36.11504007543612
+1730457260,36.89977005
+1730457280,47.847682119205295
+1730457300,45.62344892346905
+1730457320,38.532725321888414
+1730457340,38.06780115
+1730457360,40.823202959830866
+1730457380,55.47347213
+1730457400,47.55418372139838
+1730457420,27.05171255876427
+1730457440,22.85905143620574
+1730457460,20.84302905391929
+1730457480,20.84302905391929
+1730457500,19.600185344542272
+1730457520,31.32441471571906
+1730457540,23.94479973073039
+1730457560,23.94479973073039
+1730457580,23.94479973073039
+1730457600,24.61435726210351
+1730457620,25.09385894341647
+1730457640,21.990049751243777
+1730457660,23.21092278719397
+1730457680,36.9600107
+1730457700,36.9600107
+1730457720,36.9600107
+1730457740,39.02178270747027
+1730457760,38.05581519562024
+1730457780,42.37142668515218
+1730457800,35.00575529825987
+1730457820,31.41241416453481
+1730457840,30.44878245119742
+1730457860,19.513665594855308
+1730457880,34.72112080359503
+1730457900,35.33150464976249
+1730457920,45.87750692146668
+1730457940,45.30386740331492
+1730457960,39.831977597012944
+1730457980,42.00487012987013
+1730458000,43.51216261246251
+1730458020,36.43936849177024
+1730458040,48.03569050472766
+1730458060,45.58485463150778
+1730458080,41.24706671
+1730458100,36.247478143913916
+1730458120,24.23520063567739
+1730458140,23.44757380386834
+1730458160,23.44757380386834
+1730458180,23.44757380386834
+1730458200,23.44757380386834
+1730458220,23.44757380386834
+1730458240,23.44757380386834
+1730458260,23.44757380386834
+1730458280,28.897262990876637
+1730458300,30.635531378532573
+1730458320,23.4909188
+1730458340,23.4909188
+1730458360,20.08805362071231
+1730458380,26.406752411575564
+1730458400,29.591291572058232
+1730458420,32.55166932
+1730458440,37.45939361126151
+1730458460,36.35939789529772
+1730458480,39.88745226770282
+1730458500,51.56072455049796
+1730458520,42.046839041559494
+1730458540,46.68584041201257
+1730458560,41.74938378522417
+1730458580,47.33672376873662
+1730458600,44.67803781568161
+1730458620,57.07716913234706
+1730458640,59.76522971
+1730458660,46.713193752077096
+1730458680,47.55790866975513
+1730458700,58.25515947467167
+1730458720,57.77372015959965
+1730458740,49.12421446717476
+1730458760,49.02170999731975
+1730458780,54.46246798
+1730458800,64.82385186175546
+1730458820,61.304259771822046
+1730458840,61.304259771822046
+1730458860,66.18969478175254
+1730458880,65.27675028582958
+1730458900,64.56973293768546
+1730458920,64.56973293768546
+1730458940,63.322589309696475
+1730458960,58.85733377881724
+1730458980,58.85733377881724
+1730459000,58.85733377881724
+1730459020,58.85733377881724
+1730459040,58.85733377881724
+1730459060,58.85733377881724
+1730459080,58.85733377881724
+1730459100,58.85733377881724
+1730459120,50.13181910363011
+1730459140,43.99569921376252
+1730459160,54.24008447172177
+1730459180,54.24008447172177
+1730459200,61.884096675123516
+1730459220,38.58840811965812
+1730459240,35.23051815585475
+1730459260,32.13546189299312
+1730459280,36.61829227134661
+1730459300,44.67335681531202
+1730459320,34.57708139689284
+1730459340,44.97347391041569
+1730459360,37.60492824262118
+1730459380,35.749500333111264
+1730459400,44.14601473543202
+1730459420,45.186179134518625
+1730459440,48.78568101670104
+1730459460,50.70628997867804
+1730459480,61.581771212823746
+1730459500,59.85074626865672
+1730459520,62.61484566065173
+1730459540,70.34377285723785
+1730459560,69.57506327427734
+1730459580,67.27067569361476
+1730459600,76.32601125100456
+1730459620,62.16634169300074
+1730459640,58.712811171827575
+1730459660,50.82794830371567
+1730459680,57.886656431479864
+1730459700,58.43704001070521
+1730459720,52.01934209630185
+1730459740,46.38292176939035
+1730459760,49.28780617678381
+1730459780,45.35405640785154
+1730459800,51.09015104914223
+1730459820,51.78618682191056
+1730459840,81.1194324
+1730459860,72.35777930089102
+1730459880,59.33697020562316
+1730459900,50.81464312495739
+1730459920,56.11660943917054
+1730459940,55.68878593813871
+1730459960,61.48067203933889
+1730459980,60.92131809011432
+1730460000,63.42780026990553
+1730460020,63.41016949152542
+1730460040,63.43863996
+1730460060,74.05076006160853
+1730460080,65.29582015211685
+1730460100,53.41301644327759
+1730460120,61.92267088941673
+1730460140,62.33405157433795
+1730460160,53.34530138783401
+1730460180,54.57647942119992
+1730460200,55.20920502092051
+1730460220,52.59828594749014
+1730460240,54.50897714907508
+1730460260,54.50897714907508
+1730460280,64.87209382
+1730460300,53.94872499828167
+1730460320,53.48305540613233
+1730460340,49.92423198787712
+1730460360,40.12120377384478
+1730460380,46.18742327104079
+1730460400,49.10562470244168
+1730460420,55.785907859078584
+1730460440,51.326405032134545
+1730460460,63.02261593181666
+1730460480,53.66174738446615
+1730460500,64.32066492392251
+1730460520,68.19430485762143
+1730460540,67.34192756292204
+1730460560,63.751838481080355
+1730460580,66.22268470343393
+1730460600,66.18932365201022
+1730460620,73.57311941
+1730460640,58.46394984326019
+1730460660,39.74393713588945
+1730460680,36.77419355
+1730460700,50.28364431686146
+1730460720,56.327576902320565
+1730460740,42.26144971853499
+1730460760,39.67586627788703
+1730460780,42.323848238482384
+1730460800,45.78583061889251
+1730460820,57.588231324650785
+1730460840,50.281322903801296
+1730460860,42.77694098315761
+1730460880,41.42138278072655
+1730460900,38.26305425409113
+1730460920,35.5470327
+1730460940,49.56671251719395
+1730460960,40.606887146266615
+1730460980,48.10759664774263
+1730461000,42.363250496337365
+1730461020,38.74253123302553
+1730461040,38.99695019
+1730461060,53.43911740670117
+1730461080,42.70854578829288
+1730461100,42.84836485
+1730461120,47.56778577676108
+1730461140,41.66378535370998
+1730461160,43.19457757086129
+1730461180,39.83377255219947
+1730461200,59.01639344262295
+1730461220,53.78100423472474
+1730461240,53.46289022121149
+1730461260,49.25801819052178
+1730461280,51.512497473556564
+1730461300,53.89274534
+1730461320,45.43216630196937
+1730461340,41.96645929131728
+1730461360,39.87372177613067
+1730461380,49.48648648648649
+1730461400,45.04480043442846
+1730461420,51.86481755756025
+1730461440,51.256281407035175
+1730461460,40.31929347826087
+1730461480,40.01643385
+1730461500,40.47361299052774
+1730461520,47.98993265764234
+1730461540,38.71961656966792
+1730461560,43.03880440482433
+1730461580,42.47935032
+1730461600,40.69539335723466
+1730461620,38.03301237964237
+1730461640,51.77975907
+1730461660,46.25137061403509
+1730461680,41.70287887017925
+1730461700,41.074907625125974
+1730461720,41.074907625125974
+1730461740,41.074907625125974
+1730461760,41.074907625125974
+1730461780,41.074907625125974
+1730461800,41.074907625125974
+1730461820,42.02587698055767
+1730461840,50.84768659236744
+1730461860,60.49222797927462
+1730461880,51.51098901098901
+1730461900,50.609088420476326
+1730461920,62.25015041112374
+1730461940,65.75233022636485
+1730461960,70.9166895
+1730461980,58.739700121572334
+1730462000,62.49496441520075
+1730462020,66.15710093496212
+1730462040,81.02547138604037
+1730462060,74.96954108569108
+1730462080,59.53737149208113
+1730462100,53.77537212449256
+1730462120,57.54240221529942
+1730462140,63.614196694662695
+1730462160,62.45254880694143
+1730462180,61.88061921178936
+1730462200,51.548842738270054
+1730462220,51.60988017939621
+1730462240,50.813559322033896
+1730462260,55.080102642346894
+1730462280,50.16990621
+1730462300,51.93004030880645
+1730462320,62.365954002832666
+1730462340,52.01457645764576
+1730462360,38.87735326688815
+1730462380,41.20677966101695
+1730462400,40.04451639012546
+1730462420,49.41370088459165
+1730462440,48.685384773803804
+1730462460,46.665327440739254
+1730462480,51.853621902095995
+1730462500,54.2358918
+1730462520,54.68419976834503
+1730462540,51.69166610614112
+1730462560,54.86471713582945
+1730462580,64.87664253150979
+1730462600,53.63529655458124
+1730462620,59.77260422306443
+1730462640,65.73071266212061
+1730462660,56.08465608465608
+1730462680,54.156688333903524
+1730462700,56.50861181637274
+1730462720,52.87118122869175
+1730462740,52.36690450054885
+1730462760,44.4497575
+1730462780,42.982212340188994
+1730462800,38.92770668
+1730462820,47.752999389954596
+1730462840,37.27835051546392
+1730462860,40.89123662467832
+1730462880,48.89466840052016
+1730462900,42.46866223713953
+1730462920,41.227111232577215
+1730462940,43.04146829503898
+1730462960,44.714660771037714
+1730462980,46.59452147446737
+1730463000,55.68752545133704
+1730463020,65.1439853
+1730463040,60.28973452554239
+1730463060,51.99727335
+1730463080,56.08472839084387
+1730463100,45.61985756758625
+1730463120,38.894814024796695
+1730463140,34.26250946383096
+1730463160,45.782806042951016
+1730463180,49.68319559
+1730463200,43.08763473254335
+1730463220,61.00159092481151
+1730463240,53.80707221243733
+1730463260,53.859407305306696
+1730463280,56.32349907159067
+1730463300,53.62827083895333
+1730463320,53.62827083895333
+1730463340,53.62827083895333
+1730463360,53.62827083895333
+1730463380,53.62827083895333
+1730463400,53.62827083895333
+1730463420,53.62827083895333
+1730463440,53.62827083895333
+1730463460,53.62827083895333
+1730463480,53.62827083895333
+1730463500,53.62827083895333
+1730463520,56.922865392547784
+1730463540,64.91761210156672
+1730463560,56.29014241831891
+1730463580,41.53697837763393
+1730463600,41.69405642289784
+1730463620,35.80280914348664
+1730463640,42.14960953555282
+1730463660,35.398109743659475
+1730463680,44.53917986195696
+1730463700,52.39309928173458
+1730463720,45.37690570299266
+1730463740,42.964875331204574
+1730463760,47.36258404380675
+1730463780,47.66540966746658
+1730463800,39.81690237070438
+1730463820,53.80306405
+1730463840,64.72717339343032
+1730463860,68.72702974
+1730463880,68.72702974
+1730463900,68.72702974
+1730463920,65.25898592319257
+1730463940,66.21255299923541
+1730463960,64.37992929018222
+1730463980,56.31731565865783
+1730464000,48.830529339351656
+1730464020,41.35996716377069
+1730464040,41.35996716377069
+1730464060,50.54347826086957
+1730464080,47.310708584912206
+1730464100,38.87452368
+1730464120,40.13936810770584
+1730464140,41.61769789146215
+1730464160,41.16554286500619
+1730464180,49.28971451987433
+1730464200,55.455229495861545
+1730464220,39.60423634336678
+1730464240,47.72788203753351
+1730464260,43.42643287823134
+1730464280,34.35712339961863
+1730464300,48.32936372915958
+1730464320,50.472885622916245
+1730464340,35.47820209530247
+1730464360,35.75873623743418
+1730464380,41.20282378495791
+1730464400,39.804197679876616
+1730464420,52.99551879
+1730464440,53.39786110735075
+1730464460,51.05638633557415
+1730464480,53.689060028879865
+1730464500,53.85967284034068
+1730464520,58.394561560118106
+1730464540,59.828708461801995
+1730464560,51.39893544424729
+1730464580,51.39893544424729
+1730464600,45.53323865557584
+1730464620,48.90725310749898
+1730464640,52.056504599211564
+1730464660,65.63727855518026
+1730464680,63.801272505753346
+1730464700,62.85007686651962
+1730464720,71.04996281522547
+1730464740,60.438269044656444
+1730464760,61.71128750168169
+1730464780,68.48133674706914
+1730464800,59.24620390455531
+1730464820,50.960289795639405
+1730464840,55.67663284070014
+1730464860,50.62673622874178
+1730464880,63.403060883512666
+1730464900,61.579605445957206
+1730464920,49.53894852738783
+1730464940,50.87291920422249
+1730464960,47.21300373
+1730464980,49.79145299145299
+1730465000,52.593340060544904
+1730465020,49.96314907872697
+1730465040,44.515993560579545
+1730465060,38.40141371576157
+1730465080,36.49591389
+1730465100,38.75926434
+1730465120,39.88415948275862
+1730465140,51.86679431
+1730465160,40.918977705274614
+1730465180,24.58960328317373
+1730465200,20.34907048281396
+1730465220,31.60132262635805
+1730465240,38.32652240905741
+1730465260,34.155481365401656
+1730465280,40.39788331435461
+1730465300,56.25756150020164
+1730465320,31.715498357064618
+1730465340,25.81902742630463
+1730465360,29.61470146194835
+1730465380,26.69322709163347
+1730465400,24.27471469819882
+1730465420,35.96301795
+1730465440,41.15160056372056
+1730465460,46.73377156943303
+1730465480,37.84205880355809
+1730465500,36.32521634131616
+1730465520,18.15708942536552
+1730465540,31.89167340339531
+1730465560,22.87621359223301
+1730465580,22.19893163837988
+1730465600,22.678258525407742
+1730465620,31.97154471544715
+1730465640,42.91037192602127
+1730465660,28.44763387719179
+1730465680,24.66711499663753
+1730465700,25.33217017849953
+1730465720,31.56069364
+1730465740,46.71016483516484
+1730465760,22.42889145
+1730465780,24.47305997966791
+1730465800,17.87259535379085
+1730465820,18.42991913746631
+1730465840,29.937921367064952
+1730465860,37.34414801
+1730465880,36.30748573346761
+1730465900,37.241613012538124
+1730465920,39.36148579949841
+1730465940,36.137602179836506
+1730465960,31.339031339031344
+1730465980,23.07482436
+1730466000,35.32351569267675
+1730466020,31.182721501518078
+1730466040,30.03866766162404
+1730466060,20.89913312277401
+1730466080,24.61486486486486
+1730466100,20.149710785981632
+1730466120,21.89581667998934
+1730466140,19.33634585553382
+1730466160,31.235258440595732
+1730466180,28.635316438541807
+1730466200,24.64470713
+1730466220,32.42800601
+1730466240,38.10509447819991
+1730466260,38.10509447819991
+1730466280,26.24113475177305
+1730466300,26.72898462372147
+1730466320,27.463978894676323
+1730466340,39.02012248468941
+1730466360,26.025041276829942
+1730466380,26.138653768641678
+1730466400,20.47122181
+1730466420,22.27153520274359
+1730466440,23.033098209441132
+1730466460,20.19515477792732
+1730466480,32.507910304030815
+1730466500,38.14316182351373
+1730466520,35.62075821
+1730466540,38.49431818181818
+1730466560,38.92130258
+1730466580,35.57953311617807
+1730466600,27.09530811388852
+1730466620,35.065820885342056
+1730466640,35.065820885342056
+1730466660,33.46801346801347
+1730466680,23.285723955865432
+1730466700,25.51906635907179
+1730466720,25.51906635907179
+1730466740,25.51906635907179
+1730466760,21.75080558539205
+1730466780,16.20242969
+1730466800,27.495786990225817
+1730466820,22.99084435401831
+1730466840,18.29080436675373
+1730466860,22.25325884543762
+1730466880,19.27346115035318
+1730466900,32.84998329435349
+1730466920,35.021293585307426
+1730466940,37.44627388745619
+1730466960,41.80183833468505
+1730466980,33.90517986568256
+1730467000,32.64873459652842
+1730467020,34.23737916219119
+1730467040,33.82815112002675
+1730467060,35.24970194727779
+1730467080,30.88658414185346
+1730467100,45.043291496073564
+1730467120,51.91087014017456
+1730467140,45.17526322804212
+1730467160,45.81859440699459
+1730467180,47.07961061474863
+1730467200,47.76922055503917
+1730467220,56.40374331550802
+1730467240,56.03981706
+1730467260,45.63879598662207
+1730467280,49.19150073499933
+1730467300,47.44661095636026
+1730467320,47.97292724196277
+1730467340,38.47446237
+1730467360,35.97208374875374
+1730467380,38.50517183850517
+1730467400,42.92465388711395
+1730467420,39.88315089651467
+1730467440,36.56254207620843
+1730467460,36.56254207620843
+1730467480,36.56254207620843
+1730467500,36.56254207620843
+1730467520,30.14442824963562
+1730467540,39.00574907000338
+1730467560,39.00574907000338
+1730467580,39.00574907000338
+1730467600,39.00574907000338
+1730467620,38.518863530507694
+1730467640,31.89758634484598
+1730467660,22.48232626383887
+1730467680,22.48232626383887
+1730467700,22.48232626383887
+1730467720,30.05449953476007
+1730467740,32.29293338738008
+1730467760,22.66933600266934
+1730467780,19.50146627565982
+1730467800,22.96226668457097
+1730467820,15.77829405907733
+1730467840,24.09407088439881
+1730467860,31.33435277
+1730467880,31.33435277
+1730467900,31.33435277
+1730467920,31.33435277
+1730467940,31.33435277
+1730467960,38.18291781466299
+1730467980,32.10172425271287
+1730468000,31.229370158302462
+1730468020,32.03303343628307
+1730468040,20.36877932720904
+1730468060,27.60002671832209
+1730468080,29.208884117291824
+1730468100,28.27498493875092
+1730468120,28.27498493875092
+1730468140,28.27498493875092
+1730468160,28.27498493875092
+1730468180,24.31223290598291
+1730468200,26.731133351365973
+1730468220,20.74818611462424
+1730468240,20.74818611462424
+1730468260,30.72369296261807
+1730468280,18.69891906142895
+1730468300,18.69891906142895
+1730468320,15.89952569
+1730468340,21.89169089311068
+1730468360,14.39248627554732
+1730468380,25.645086781495408
+1730468400,31.27770793356713
+1730468420,18.038611073870488
+1730468440,20.64546118600173
+1730468460,18.00520729020629
+1730468480,27.62716413904174
+1730468500,23.80474757606152
+1730468520,28.192867105001003
+1730468540,21.72571121846484
+1730468560,16.77972865123703
+1730468580,26.393354769560563
+1730468600,41.06382978723404
+1730468620,31.22025383395029
+1730468640,34.79713922866119
+1730468660,21.36264028
+1730468680,19.795768537676032
+1730468700,27.922897972748416
+1730468720,20.09895694035838
+1730468740,34.692136011553096
+1730468760,29.003800217155266
+1730468780,24.76381909547739
+1730468800,34.34516362678607
+1730468820,38.059901056290954
+1730468840,34.57317073170732
+1730468860,35.12385568120625
+1730468880,32.436213443474784
+1730468900,26.47398843930636
+1730468920,28.01961574633884
+1730468940,21.724941724941722
+1730468960,20.97009966777409
+1730468980,21.66835528537656
+1730469000,34.624907983671285
+1730469020,32.32847541
+1730469040,24.01372212692967
+1730469060,20.51264846
+1730469080,22.819283526434937
+1730469100,23.81901534781742
+1730469120,37.191018722100075
+1730469140,36.515957446808514
+1730469160,23.21085733200927
+1730469180,23.21085733200927
+1730469200,23.21085733200927
+1730469220,21.602393617021278
+1730469240,26.68438538
+1730469260,22.07543430873902
+1730469280,16.93769658033862
+1730469300,18.21992271923505
+1730469320,30.29649947753396
+1730469340,19.679913196799127
+1730469360,21.61732565070534
+1730469380,21.61732565070534
+1730469400,21.27602953895283
+1730469420,21.63648558014755
+1730469440,24.34577867473932
+1730469460,16.91942502967163
+1730469480,39.462151394422314
+1730469500,24.85868102288022
+1730469520,22.57001647446458
+1730469540,18.96175946106598
+1730469560,20.47909816815406
+1730469580,19.03401541323412
+1730469600,23.94974740760436
+1730469620,36.95998936594444
+1730469640,35.21672343551726
+1730469660,34.187579045463615
+1730469680,32.12974704665287
+1730469700,27.31086841
+1730469720,38.35762294536501
+1730469740,29.30304456
+1730469760,19.42186680650236
+1730469780,21.65031581776643
+1730469800,18.45777395674889
+1730469820,20.12511646479436
+1730469840,25.592766242464844
+1730469860,28.40485074626866
+1730469880,18.89125232650891
+1730469900,24.686847599164928
+1730469920,20.21905657841688
+1730469940,28.15179105469529
+1730469960,22.88437017649822
+1730469980,27.38215318404949
+1730470000,20.26686139139671
+1730470020,32.91257597007948
+1730470040,30.89622641509434
+1730470060,23.608611611011128
+1730470080,22.49634259874983
+1730470100,22.58958622055039
+1730470120,33.69702614596501
+1730470140,33.69702614596501
+1730470160,32.09868244151654
+1730470180,35.666912306558594
+1730470200,32.77645186953063
+1730470220,32.04888473353813
+1730470240,32.912730395719095
+1730470260,40.16941301038978
+1730470280,38.41577540106952
+1730470300,36.18870678787075
+1730470320,38.76273556635813
+1730470340,55.75916230366492
+1730470360,50.75052639
+1730470380,48.83176005891411
+1730470400,49.75300400534045
+1730470420,58.64484407624436
+1730470440,54.049001936043794
+1730470460,64.36445743126576
+1730470480,46.32503062474479
+1730470500,56.23274161735699
+1730470520,53.762079306897704
+1730470540,46.85549365729244
+1730470560,44.75849973
+1730470580,33.08661732346469
+1730470600,33.08661732346469
+1730470620,33.08661732346469
+1730470640,38.01349275265513
+1730470660,36.14620633740963
+1730470680,36.14620633740963
+1730470700,36.14620633740963
+1730470720,42.91154462854089
+1730470740,44.07005393886928
+1730470760,37.81862087469815
+1730470780,37.81862087469815
+1730470800,37.81862087469815
+1730470820,33.980839114634946
+1730470840,36.30320934604539
+1730470860,51.25967720771551
+1730470880,36.73871669385536
+1730470900,41.08848916123081
+1730470920,37.84180338802188
+1730470940,35.68517404957573
+1730470960,31.49969885565148
+1730470980,35.84502338009352
+1730471000,33.00080342795929
+1730471020,38.48350043157825
+1730471040,38.32463535394085
+1730471060,40.38901601830664
+1730471080,40.38901601830664
+1730471100,40.38901601830664
+1730471120,40.38901601830664
+1730471140,40.38901601830664
+1730471160,40.38901601830664
+1730471180,48.87621375704488
+1730471200,52.569169960474305
+1730471220,47.09051007514797
+1730471240,58.33610207
+1730471260,53.54277165782674
+1730471280,40.26438254535644
+1730471300,36.93499494779387
+1730471320,32.44394171
+1730471340,32.44394171
+1730471360,32.44394171
+1730471380,32.44394171
+1730471400,46.01247568582735
+1730471420,40.06772773450728
+1730471440,30.052712350703942
+1730471460,30.052712350703942
+1730471480,35.18879989230666
+1730471500,38.243739565943244
+1730471520,44.189033871949164
+1730471540,34.81046096948884
+1730471560,47.20442436096311
+1730471580,42.51197948302625
+1730471600,38.54737404784177
+1730471620,37.91305525460455
+1730471640,36.10758011641132
+1730471660,37.039045553145336
+1730471680,43.15297261189045
+1730471700,38.853503184713375
+1730471720,35.17715353306199
+1730471740,34.89333333333333
+1730471760,36.4681366
+1730471780,47.789981248325745
+1730471800,50.704129532621266
+1730471820,48.58399413606983
+1730471840,50.54161340240867
+1730471860,42.66765920010828
+1730471880,47.63835394283398
+1730471900,55.947756195579366
+1730471920,40.381515314347126
+1730471940,37.71941668917094
+1730471960,36.63983903420523
+1730471980,39.27550681289465
+1730472000,44.166045913184796
+1730472020,42.391963864356505
+1730472040,30.46038543897216
+1730472060,35.610406937958636
+1730472080,26.141911322142708
+1730472100,20.298746332355293
+1730472120,31.26879134095009
+1730472140,28.251121076233183
+1730472160,25.16595857673925
+1730472180,20.47244094488189
+1730472200,23.87831180934629
+1730472220,37.09839694144476
+1730472240,35.42522389064709
+1730472260,24.06247896047936
+1730472280,28.891257995735607
+1730472300,26.25462229265716
+1730472320,31.15516892575359
+1730472340,22.915549597855232
+1730472360,18.08574677279112
+1730472380,20.33028488716363
+1730472400,24.083769633507853
+1730472420,22.035124646451358
+1730472440,26.93534161069272
+1730472460,34.10205434062293
+1730472480,36.0267798
+1730472500,37.23931798111541
+1730472520,34.47232125
+1730472540,35.82802547770701
+1730472560,41.32608549761761
+1730472580,36.17207486199004
+1730472600,32.578733604101465
+1730472620,32.19418504536724
+1730472640,32.19418504536724
+1730472660,32.19418504536724
+1730472680,32.19418504536724
+1730472700,32.19418504536724
+1730472720,39.36690457309014
+1730472740,44.14117647058824
+1730472760,41.21171770972037
+1730472780,38.58634217700645
+1730472800,34.13351314902225
+1730472820,38.34731655346917
+1730472840,46.93023883416543
+1730472860,46.23663101604278
+1730472880,40.03473149879776
+1730472900,38.6596548
+1730472920,37.68770764119601
+1730472940,36.97176046465051
+1730472960,38.74873182279337
+1730472980,40.929888305746196
+1730473000,20.40816326530612
+1730473020,22.49581799933088
+1730473040,30.79198159710812
+1730473060,27.20874111811235
+1730473080,23.46306068601583
+1730473100,32.06847745501112
+1730473120,25.13432053466125
+1730473140,25.58517115410635
+1730473160,28.49224678323985
+1730473180,28.404433993953642
+1730473200,26.68164398044139
+1730473220,36.97305229
+1730473240,35.038873014818265
+1730473260,33.93955783922656
+1730473280,34.64409490802453
+1730473300,34.41190206151178
+1730473320,27.013772253946932
+1730473340,26.579765764573548
+1730473360,19.936457505957108
+1730473380,23.63252933
+1730473400,22.977368207074132
+1730473420,35.26058085720035
+1730473440,23.64499538
+1730473460,23.04955015
+1730473480,23.04955015
+1730473500,23.04955015
+1730473520,23.04955015
+1730473540,23.04955015
+1730473560,23.04955015
+1730473580,23.04955015
+1730473600,23.34434897554527
+1730473620,22.448707700506258
+1730473640,22.448707700506258
+1730473660,22.448707700506258
+1730473680,22.448707700506258
+1730473700,22.448707700506258
+1730473720,28.209897044171374
+1730473740,22.66862758205762
+1730473760,30.29136523528241
+1730473780,23.64253393665158
+1730473800,35.020730239400834
+1730473820,15.26242637
+1730473840,15.26242637
+1730473860,19.38809444629199
+1730473880,20.70224346210009
+1730473900,30.03630984400215
+1730473920,27.27943148297131
+1730473940,26.20588431516445
+1730473960,26.779999999999998
+1730473980,40.23302263648469
+1730474000,35.55763573147901
+1730474020,34.06512745
+1730474040,36.28096995255667
+1730474060,36.00943714189417
+1730474080,29.27350427350427
+1730474100,30.434201736806948
+1730474120,19.78631051752922
+1730474140,20.561630889698932
+1730474160,19.86618972
+1730474180,34.606096207819334
+1730474200,25.517897167696802
+1730474220,21.29851143009038
+1730474240,23.62911266201396
+1730474260,27.19222318396698
+1730474280,42.415504291845494
+1730474300,36.84175038770144
+1730474320,35.95824800158552
+1730474340,35.313066237165295
+1730474360,35.555555555555564
+1730474380,40.91728419787703
+1730474400,21.94044831047173
+1730474420,15.680551870522692
+1730474440,15.680551870522692
+1730474460,15.680551870522692
+1730474480,15.680551870522692
+1730474500,15.680551870522692
+1730474520,15.680551870522692
+1730474540,15.680551870522692
+1730474560,18.95781968
+1730474580,20.920225091029458
+1730474600,18.104076897580377
+1730474620,18.104076897580377
+1730474640,18.104076897580377
+1730474660,18.104076897580377
+1730474680,18.104076897580377
+1730474700,18.104076897580377
+1730474720,18.104076897580377
+1730474740,18.104076897580377
+1730474760,28.48270335247763
+1730474780,21.39065817409766
+1730474800,21.39065817409766
+1730474820,22.15596944928313
+1730474840,27.389292795769993
+1730474860,33.09257173149559
+1730474880,33.09257173149559
+1730474900,33.09257173149559
+1730474920,33.09257173149559
+1730474940,33.09257173149559
+1730474960,33.09257173149559
+1730474980,33.09257173149559
+1730475000,33.09257173149559
+1730475020,33.09257173149559
+1730475040,33.09257173149559
+1730475060,33.09257173149559
+1730475080,33.09257173149559
+1730475100,33.09257173149559
+1730475120,33.09257173149559
+1730475140,33.09257173149559
+1730475160,33.09257173149559
+1730475180,33.09257173149559
+1730475200,33.09257173149559
+1730475220,33.09257173149559
+1730475240,33.09257173149559
+1730475260,33.09257173149559
+1730475280,33.09257173149559
+1730475300,33.09257173149559
+1730475320,33.09257173149559
+1730475340,33.09257173149559
+1730475360,33.09257173149559
+1730475380,35.466191230850505
+1730475400,35.38969795037756
+1730475420,34.33124416588879
+1730475440,34.33124416588879
+1730475460,34.33124416588879
+1730475480,34.33124416588879
+1730475500,34.33124416588879
+1730475520,34.33124416588879
+1730475540,34.33124416588879
+1730475560,34.33124416588879
+1730475580,34.33124416588879
+1730475600,34.33124416588879
+1730475620,34.33124416588879
+1730475640,34.33124416588879
+1730475660,34.33124416588879
+1730475680,34.33124416588879
+1730475700,34.33124416588879
+1730475720,34.33124416588879
+1730475740,34.33124416588879
+1730475760,34.33124416588879
+1730475780,34.33124416588879
+1730475800,34.33124416588879
+1730475820,34.33124416588879
+1730475840,34.33124416588879
+1730475860,34.33124416588879
+1730475880,34.33124416588879
+1730475900,32.78601342817257
+1730475920,46.3142418
+1730475940,46.3142418
+1730475960,46.3142418
+1730475980,46.3142418
+1730476000,22.76794725156429
+1730476020,17.417978127500668
+1730476040,17.777480967009478
+1730476060,17.777480967009478
+1730476080,17.777480967009478
+1730476100,17.777480967009478
+1730476120,17.777480967009478
+1730476140,17.777480967009478
+1730476160,17.777480967009478
+1730476180,17.777480967009478
+1730476200,17.777480967009478
+1730476220,17.777480967009478
+1730476240,17.777480967009478
+1730476260,17.777480967009478
diff --git a/atsc/tests/csv/cpu_utilization_no_headers_only_values.csv b/atsc/tests/csv/cpu_utilization_no_headers_only_values.csv
new file mode 100644
index 0000000..accc16d
--- /dev/null
+++ b/atsc/tests/csv/cpu_utilization_no_headers_only_values.csv
@@ -0,0 +1,2854 @@
+60.486139011651275
+65.44663767146203
+68.66631313800875
+60.99319727891156
+56.17527173913043
+53.807848084456545
+52.31012876127522
+35.62985477879095
+37.19002740458525
+34.84838298
+33.60759917051308
+44.19748097258705
+36.09307359307359
+34.67887455724119
+35.71475909603022
+48.76137344756592
+37.707573916565416
+34.58996493121122
+46.48191956742143
+44.41222094827005
+46.316139767054906
+47.09473966720344
+36.95083506606748
+35.42657998793647
+48.01377118644068
+38.17094713955701
+34.37643207855974
+26.11023405861761
+21.30729481
+23.96445750935329
+37.30611696770295
+43.96847848050111
+36.69335845501649
+37.55643911314779
+36.19338557577368
+23.81870140227288
+35.65270106425973
+22.84139214
+26.70439234707626
+21.02712526081982
+21.53653224209623
+21.156947057242682
+25.81049851436118
+22.219263795513548
+21.96208846025939
+22.53635992931902
+19.77565160013197
+21.95712077424558
+27.08235766374523
+25.58124174372523
+20.18625217740855
+20.94590087385765
+21.25181950509461
+22.14171762829035
+37.76344657934978
+45.50954557676795
+41.231195566112426
+46.377883903948344
+34.29653248306098
+32.19118142039717
+37.315024663378225
+25.59994681911853
+20.73656270736563
+27.43291950340408
+20.63555114200596
+18.21381142098274
+30.620569747224817
+21.22963741460851
+20.70415846957466
+21.64493480441324
+23.065456023851468
+28.92965553649202
+25.725806451612897
+21.142473118279568
+19.79298831385643
+22.04687708083633
+26.79917294737544
+29.436450839328536
+24.18104025
+40.60710957262681
+20.35392229981694
+22.03627150286705
+32.17867511
+26.096299778389632
+19.43680858196447
+20.32745758235954
+19.039746048541762
+24.4175898
+34.222784136278946
+34.10919731329388
+38.467752793860235
+38.22957850298197
+41.51776615259189
+46.236056375659615
+48.114347067507715
+39.23056443024494
+50.03657158055722
+52.31752438614195
+36.55439721999465
+36.15706597568268
+39.24971630732261
+36.99751694517147
+33.29335731894197
+43.38666666666667
+42.50584697627798
+42.986916141278556
+31.782282479926288
+22.66353572626635
+27.19292367
+20.786928976325438
+34.87138155685405
+40.84005376344086
+39.432387312186975
+36.17666891436278
+37.57763975
+42.70142491299494
+21.41298417894163
+22.276476410425598
+31.93790220924832
+27.70429060632512
+22.47681763
+22.26557244719494
+35.286311459093625
+26.052968528876608
+30.853682879765792
+32.66666666666667
+24.129731157936458
+24.15926822706484
+25.447329476658147
+26.376695929768562
+29.097347497828558
+22.50368583299826
+20.786367855722048
+29.71688641991835
+26.363757317722193
+25.14576271186441
+35.30390786232608
+40.011906336817034
+39.39886271324127
+39.47792242651993
+44.135122504840105
+40.99169087347159
+32.38830441
+24.74517167381974
+27.41140735740803
+36.91883902176834
+25.10621348911312
+27.30720606826802
+26.944444444444436
+25.381358779651908
+38.260459475847206
+41.15951742627346
+32.933647692825865
+29.83725136
+28.263221966725283
+35.587472571314585
+34.71244521184752
+41.03668798384382
+29.36147475287203
+26.94425789120215
+40.01467938880363
+49.44186046511628
+53.80097218
+49.68632371392723
+52.64186548052297
+50.88970343218927
+38.34000135620804
+43.223908537808384
+49.81630153762417
+27.00262750117901
+27.50083194675541
+27.219538744132247
+31.68099140515691
+39.55527933153788
+26.05814739688979
+26.094219846308047
+21.58046555309586
+34.84900129524848
+31.306502377285213
+26.586470230452118
+25.58685446
+27.18628632650328
+36.843167597389495
+39.59525756336877
+49.02453946026783
+49.18165285916347
+33.493249263244465
+34.78144811478145
+26.324659565862206
+25.61658445540595
+30.28532608695652
+21.19064962585259
+21.207874541207868
+32.79602391026933
+35.82693085321472
+41.37585238668271
+37.07464694014795
+37.21776639482375
+34.132013201320134
+46.734555329260004
+46.84457982915039
+39.53835709436524
+36.65958012224289
+36.50531598758502
+35.36992840095465
+30.10795681727309
+44.76462386295731
+37.69514237855946
+48.81235154394299
+46.58610271903323
+51.05228325612177
+48.15459843258088
+52.371202113606344
+61.458474004051325
+49.65328710494733
+49.87977558108469
+54.95854063018242
+63.021640935751364
+53.21765722643266
+61.60167037111874
+35.57996899642785
+37.143428419088245
+32.586287158342756
+41.311961206896555
+48.73438856608562
+41.78422412
+36.427850655903136
+37.19288900725747
+42.96404211641613
+57.68622280817402
+53.098124195842075
+52.26154364678126
+49.01474530831099
+54.05278670953913
+45.73539288112827
+35.32377919320594
+35.006578947368425
+46.03695273798367
+36.82475613858056
+37.13409348536374
+39.27639938473885
+49.68041445199489
+42.20006646726487
+40.17384218389244
+38.915694947058036
+41.75581706338593
+52.075165607678784
+50.456940831165376
+40.57173533753859
+35.02558577969297
+40.18215912135012
+52.95612627073301
+58.64218216
+53.43331764072903
+48.316712402114995
+50.83450633420471
+54.37479270315091
+63.553834857486144
+46.21854485776805
+42.85230810883985
+47.26942970376839
+41.207526736158115
+35.52765524261961
+38.02713848609818
+33.76570733684637
+31.45962525838501
+52.80914069366507
+36.13241825
+33.04115088658414
+35.042111545858475
+34.93238242615892
+32.870122928915016
+42.060397806119084
+43.20820039372751
+39.036823395264605
+35.851851851851855
+38.36165873555404
+45.68090787716956
+39.85813667985268
+36.00215299737603
+43.21259422320059
+34.65829380720155
+36.57957244655582
+33.11640879208447
+31.45511255763493
+32.82703245232519
+31.44505638604773
+21.522111758656152
+22.13945372793772
+28.923180140892317
+34.66495714960524
+29.248699545664046
+23.60378634212306
+18.13502276776876
+23.09821127
+18.46430234879233
+21.980595516895278
+19.96267413
+30.97084782
+25.10418066944482
+20.00936266969839
+35.34952566776559
+29.81247100921079
+25.17355260497405
+21.169099513430652
+34.92754565
+37.17080511662904
+21.409503470368392
+23.25921981094395
+20.97596504005827
+22.72696727370472
+19.84461857879579
+31.703260146373918
+33.055720801110446
+33.32668660684613
+34.97022813942597
+35.15289699570815
+48.34773431831673
+37.36874195515209
+35.503077334760505
+39.11019485269668
+52.91345898593045
+50.37819420783646
+45.80228845130389
+32.78732366
+33.295401298614365
+29.578305264553773
+36.18214595713427
+40.21753726332751
+33.70913942628419
+36.16545934601584
+31.95348837209302
+37.49916683330001
+42.32495812
+33.68399757
+21.11274378840502
+23.019422011613162
+29.15920100869334
+45.94486081
+47.07108749747797
+33.54696575205287
+31.73411953899759
+41.02359056377449
+22.828567592965502
+18.17155006311873
+30.88410991636798
+35.68500974265941
+37.12748065926673
+34.73145440042542
+34.767971483266216
+37.06009745533297
+27.660146534919672
+32.5992369
+22.658834649922362
+21.70087976539589
+21.788379530916842
+22.53679139444334
+27.55558523068705
+22.13990121479108
+34.60972017673049
+19.24048949188614
+39.57487922705314
+37.26411880918319
+34.699343779295575
+21.359417773920008
+25.20244914
+27.68452983
+29.75707154742097
+24.3795329
+22.274060250599838
+34.10439708659293
+29.122807017543863
+20.93487814622453
+19.174645097391878
+31.70421024403325
+24.08554867079752
+30.84798275745942
+30.50656411616496
+42.838770813591445
+34.74747474747475
+37.83599399645245
+45.82115333510497
+39.386307906421905
+39.94904116937106
+45.51403130059363
+26.68790672451193
+20.86910579844142
+21.21068650659452
+20.01704694
+30.84603607801371
+23.44431232659532
+23.96895038812015
+19.38721254818556
+34.82012478428249
+32.57348530293941
+26.91490774
+21.863701480319257
+22.71247258954083
+35.23923763827802
+37.49750548792656
+19.69717362
+20.56539916384631
+24.81531229
+37.84676354029062
+33.83318956937164
+32.76707177868797
+35.17828632251197
+34.98411016949153
+44.183845893883074
+35.24345809870818
+24.30740543420352
+30.76513285
+51.190875545119084
+39.240676147035146
+31.91475235692471
+33.22814556197126
+34.89415023412254
+30.22724216754068
+18.15109343936382
+20.97943974981702
+22.99775696002111
+29.18175235336713
+32.909613969358396
+22.64340825226434
+21.18661054319503
+22.67031062524997
+26.69237930575874
+34.66869060190074
+25.19895673109075
+13.85002983491348
+19.43430656934307
+18.32623538159941
+22.059809574896068
+25.27326929446837
+22.99009900990099
+17.997584216883638
+21.049823250850398
+29.932468220338983
+20.52
+20.31937964119077
+41.424332344213646
+23.54715463502916
+18.20072454045351
+21.881458763917593
+21.53169707310671
+18.249883013570432
+23.32113144758735
+29.97060395510422
+34.33513657917585
+34.91112442580387
+35.285406569736125
+34.29829242
+44.45765622935659
+53.20042672356314
+40.45776058
+38.07283967208709
+34.79069767
+50.84262208067941
+41.31408328320299
+31.22733986840338
+31.797909871530322
+40.900039824771014
+44.88556749182625
+34.01611820013432
+34.40860215053763
+36.75643485740753
+32.50166333998669
+44.98946536739531
+46.733119410118114
+24.14351227407607
+24.68490211853044
+22.32809168443497
+23.16928354664204
+33.22167883700677
+29.96464545393903
+28.36519182
+21.176784167388547
+16.30019749835418
+20.015765617815152
+27.953852274234194
+35.46945852227261
+36.41063515509601
+38.69091395333289
+34.19411093970085
+28.967475826627897
+35.70355860267711
+22.36564984034241
+24.60502633157789
+23.37397034596376
+22.41693064635708
+29.26583975863225
+39.94639866
+36.33333333333333
+33.03994132159765
+38.57947844319224
+40.53142702994335
+36.56751493990465
+43.76506830967051
+50.48543689320388
+48.98703404
+49.71836520827875
+52.44043174258367
+49.91096748664512
+51.367468640756734
+46.47172708458509
+35.58664078709591
+30.059069489613062
+33.61670794564563
+33.88996075
+35.542728237791934
+48.30723939515589
+36.79771550006641
+39.08222563623004
+44.96476009758742
+35.766666666666666
+31.08512193501229
+27.06913996627319
+23.63337713534823
+22.93230540339487
+20.32438926712054
+31.04180491041805
+30.34334475576161
+22.11078639744952
+17.69195036166965
+20.69943912900033
+29.47957619777437
+24.26090414954262
+24.88802727455044
+32.60811173784582
+38.670654702558984
+36.53769445552453
+38.686773059237225
+44.49394866338609
+39.43362361231137
+36.670010030090275
+39.78393612024425
+50.367136336574724
+54.624664879356565
+48.82478632478632
+48.72632702053881
+46.63865546218487
+37.27703921437638
+26.413065394179718
+30.469637250317323
+33.98622902600441
+22.457942030943858
+24.41049485220857
+23.73957919809448
+27.723898067018933
+33.299839228295816
+27.600905821233518
+28.06345009914078
+26.582709326072163
+22.93351152689609
+31.98318430535166
+27.673671529915673
+26.77456181586193
+26.05605471369183
+39.60537021969081
+41.43482654092287
+37.40596453519613
+41.39081687132942
+55.06904681711015
+44.09576626538618
+36.537683699609005
+46.23255813953488
+42.189388851578244
+36.65456501277397
+34.58524723047825
+35.52454153182309
+49.191472510065346
+51.28568536140919
+38.10852095370924
+37.74879423952177
+37.00243704305443
+43.84137792910074
+49.41121370266292
+38.83952255
+34.38598863943738
+34.06535603918732
+51.61354581673307
+26.97154471544715
+15.350671926188411
+29.97287821657736
+23.08156289792909
+22.2304883
+23.61259338313767
+20.12629316135967
+19.54524423293013
+28.948069241011982
+46.901530850992714
+37.79516934286871
+34.73642278581862
+33.676975945017176
+41.59095497678175
+34.73620212119165
+23.7945982
+20.90604026845638
+21.040299906279287
+27.41170268845546
+37.530479544838805
+23.46870392708752
+27.52132124001624
+25.26709401709402
+32.73733216
+23.190921228304408
+26.41496825927163
+38.582357321779156
+27.435587761674718
+27.83457249070632
+21.46809787319155
+20.401292755184492
+20.87394518218094
+28.349892588614388
+21.388741370154012
+22.161983471074382
+31.26464484166834
+35.051338836319715
+44.17202306010205
+45.07705767548287
+37.56539235412475
+35.79571971464764
+39.99867444326617
+49.03717571543193
+49.46928486135067
+47.58528428093645
+43.49206349206349
+47.73498304
+51.20341356090407
+54.26039078761834
+21.66666666666667
+26.950023353573098
+18.52466635681562
+32.65210084033613
+38.59954479
+33.41099720410065
+43.33244610061219
+45.153789942204206
+36.27246415830921
+21.46649640432825
+22.25023218787316
+20.161073825503358
+22.83053757759873
+18.57543765869304
+24.13815876370361
+35.24880708929789
+32.87774208979703
+25.25266046449368
+33.69115882117616
+22.95028630921395
+23.60029788098301
+28.96965655218406
+39.02143961287721
+37.65250033516557
+31.318463771941403
+32.69749258560259
+30.563559605977346
+22.689132040008058
+16.51907583727279
+24.18043710021322
+31.11972207375735
+21.55712571657112
+30.87953479151523
+26.18806192117894
+23.29117330462863
+20.490261920752182
+41.72652211854736
+35.274842851410995
+43.77011802575107
+36.90940170940171
+34.68679043675705
+30.932458665238638
+39.73851030110935
+38.59073875802998
+33.31976665309999
+31.121685927940177
+24.94380536824012
+31.71501477
+36.10472195913949
+37.22351378279641
+37.091503267973856
+35.33704883227176
+39.81866521235258
+46.70972951150585
+43.77892730737816
+34.98701471665446
+39.60668501241694
+31.48490043518396
+31.48490043518396
+41.02271200107512
+45.88093806374023
+45.88093806374023
+35.089610564613935
+35.81292948161985
+34.42698100851343
+32.201309328968904
+34.779100262255405
+34.76925131904094
+45.32049153105281
+53.57094821788837
+45.45333864
+41.32019438444924
+45.08928571428571
+41.98363338788871
+62.30625334045965
+58.43305785123967
+57.33758307337583
+60.8169239
+59.14755402359803
+60.43747959516814
+67.70488480508075
+60.63496298272527
+59.42711135018827
+59.42711135018827
+61.44005358338915
+58.411494711634404
+55.71884559116904
+59.04679606487636
+57.547611092549275
+59.91642345449721
+67.12928653237216
+33.557449074690446
+34.60714763753179
+34.60714763753179
+32.67214319107727
+28.482992744458503
+28.482992744458503
+31.182147165259348
+39.10188379
+40.39721678038235
+41.64715327490466
+40.869623746382175
+41.95261166788345
+41.14308877
+35.11280167890871
+43.067966422962364
+35.36811440677966
+40.581567939549316
+49.891642963564955
+45.613220069020436
+45.919470341845695
+53.94454862041253
+42.43100649350649
+38.77878950187467
+47.2276232
+32.68588564029169
+40
+35.94150155007413
+43.35788325150027
+44.00269541778976
+36.109054190508246
+34.29368029739777
+38.58134987526128
+50.932798395185564
+46.090007421901355
+32.35489440828795
+32.923665164550044
+37.30455699585728
+36.69993346640053
+35.66580697705986
+33.71920850720151
+30.148084202138257
+31.691042663611242
+34.56404807756159
+36.03460801796447
+35.29649868447683
+44.37909370405026
+40.19509587759738
+26.01365187713311
+22.97784303256807
+32.05024855568991
+36.45581520279345
+35.14989920010405
+40.629836484758755
+39.97693507903127
+47.92586645256256
+28.34533351446814
+25.19881385631487
+28.869901875709232
+33.29504763191676
+22.12366121295749
+27.31948298030723
+24.63671510465271
+30.10694827470236
+29.93750419998656
+19.56696070569366
+16.36303630363036
+29.365184988022357
+27.17894458404973
+24.61590070446159
+20.66599394550959
+31.03586188508199
+31.88328912466844
+25.41496598639456
+26.10714766455781
+24.67326732673267
+27.57547042100988
+45.90098482832047
+31.053129647154247
+21.58757664622767
+21.11368909512761
+20.585483224167888
+27.528275679608438
+21.86555638
+29.72882936904524
+21.55401809473124
+28.81627209
+37.70788626609442
+34.255529877847465
+35.05364661143556
+37.22627737226277
+40.57863999462979
+37.66190188577948
+35.88813402809841
+36.098609525088996
+43.95530875761022
+49.36181469479532
+49.22664881151657
+45.90822440087146
+51.16648992576882
+46.92813113865745
+59.59981262129425
+62.45946124826261
+51.89856343157753
+51.449420901118025
+47.26509612780937
+51.856319528213376
+51.59321124304018
+73.33952008484688
+60.616136255682974
+49.69623329283111
+46.011576076109364
+49.780941949616654
+42.17691637046476
+37.91560956068901
+33.27280065
+44.853236831523915
+46.12977826146103
+36.30760986066452
+37.19086021505376
+37.363080438142596
+35.79670512906023
+47.22148009617954
+50.08126777732629
+46.19241826569499
+47.40407791023386
+50.67929188966653
+44.51406479136115
+39.38476036618201
+38.62369101047192
+34.28629086506121
+36.232664602127365
+31.58320960993386
+45.22458786624841
+39.28138061210732
+42.51767081790643
+34.34208772163712
+33.88624001074475
+37.9434998
+41.10879364865774
+42.89370874309578
+44.129419613075385
+42.618290123874644
+31.82182449579271
+44.55917394757744
+29.52880285003697
+45.54886211512718
+38.87951400607492
+36.72069406147017
+32.28084470700075
+33.78178393649939
+48.87043189368771
+48.77589454
+40.69610879224451
+35.39546872880206
+34.339747843397475
+42.27260531258385
+36.18319586937571
+32.66088359895098
+33.47785485592316
+36.52971092077088
+41.89736664415935
+41.89736664415935
+49.08722109533469
+36.29191321499014
+38.32791884903534
+38.32791884903534
+45.27709071461996
+46.22025832766825
+38.90647094471753
+38.47301328993762
+38.75397415950754
+40.073279956574844
+45.59799539482595
+38.03548539431964
+38.14989871708305
+41.73942243116185
+42.316623412102174
+39.54475048824837
+53.430407975862515
+52.61909602222674
+54.58445040214477
+64.06462123269074
+64.06462123269074
+53.687971952535065
+51.807802093244526
+51.807802093244526
+51.807802093244526
+51.807802093244526
+51.807802093244526
+51.807802093244526
+51.807802093244526
+49.52978056426332
+56.59004352557127
+51.254264499297605
+48.10083806434171
+51.04173645072687
+49.97971602434077
+56.163734776725306
+57.738896366083445
+58.66365441226809
+39.18223666395884
+35.34442440034743
+42.08784389436668
+37.268722466960355
+41.18979415095597
+49.40983830251757
+51.09665302837212
+39.22090326101787
+35.84058759521219
+36.421832884097036
+44.18899953217938
+44.08376253727297
+44.08376253727297
+44.08376253727297
+44.08376253727297
+44.08376253727297
+44.08376253727297
+44.558507783145465
+45.25078584119175
+44.254858148825235
+51.3671875
+44.62227045
+40.969044414535674
+41.304048471495456
+40.13157894736842
+53.83052225157488
+40.16890281277668
+37.31045490822027
+34.58672086720867
+58.986673882161945
+66.96249833177632
+61.98671096345515
+64.63950266909926
+65.49518615767857
+79.50915465523958
+74.15551085524034
+61.548378341701714
+53.856989535819686
+62.40905416329831
+58.72414022483869
+55.830243098475485
+52.474029524330234
+52.15139442231076
+51.72295442
+48.50563782094824
+50.77615751629595
+63.10699173886762
+57.09787633147987
+58.347849390325614
+58.347849390325614
+58.347849390325614
+50.26401978477375
+51.13435674500747
+53.36010709504685
+59.18435233850043
+51.35725003410177
+48.771013327975346
+56.349259086982514
+54.25416610215418
+44.59104678559408
+54.02556331532481
+57.82784129119032
+58.96522140830199
+50.74385728710872
+49.284898945813474
+47.003367
+49.03405696076479
+60.076252723311555
+57.54885723749587
+50.57633025063664
+50.06292640922037
+46.546326517018585
+49.10665575559193
+57.273144806771214
+60.806255898611305
+53.83994076866124
+44.64164747324211
+46.507654905052895
+46.38795986622074
+46.96031043638097
+52.05990016638935
+58.092061374249504
+48.29733938121996
+50.39018208497299
+48.04585372393913
+53.669724770642205
+56.69660812441346
+54.399679294447786
+53.80830412510112
+53.80830412510112
+53.80830412510112
+53.80830412510112
+46.89854682454252
+41.68616701210136
+39.62634535977797
+36.40274230407313
+35.70464362850972
+35.50093758371283
+47.88928713659896
+47.13050582068214
+37.40411377367456
+43.45424292845258
+39.35300871209563
+35.43980079
+45.00235579188261
+44.42262106332543
+50.08814755
+48.08102345415778
+49.88662131519274
+54.26648721399731
+46.71300893743793
+54.43674412284278
+41.98303715670436
+37.63788001076137
+40.42738478027867
+39.89651416122004
+45.88976800321845
+46.354976015134106
+41.98835886800027
+52.548625083836356
+39.53504043126685
+39.17806355038443
+42.34547276979619
+49.05357501
+49.52192323
+48.31483081832081
+59.736397431564725
+49.52916469073911
+45.15845665247652
+37.93890567304465
+41.06365614798694
+44.62981574539363
+52.2682976
+38.31756618798067
+39.64472878241339
+47.48674585598282
+55.30825425001707
+42.91596355045562
+36.13371304583672
+39.73980686695279
+34.43636610722294
+44.582521663196076
+64.81700895208003
+57.595149882115194
+62.34407128169978
+56.451502965033065
+54.55348647546554
+63.33223358627515
+66.24126813541108
+67.91932773109244
+63.890537421694695
+59.87578487578487
+52.167756725777096
+59.85445724681625
+41.75672884972809
+35.11024468943264
+37.678180733275866
+40.48305201042953
+50.41032428855063
+61.91760501651945
+66.31117111036112
+53.39262852907486
+58.89763779527559
+56.19967793880837
+56.524941412788756
+63.80490398033092
+52.663170967296814
+51.07851018220793
+51.76919983916365
+47.57967729030537
+61.64134964376932
+65.09258025842547
+66.61782336485946
+63.78699031774172
+67.38246082027342
+72.85903375
+75.47132757
+78.25263727346497
+67.71917808219177
+65.24618678084025
+66.93850267379679
+70.32138606004497
+61.59459459459459
+65.10224606101241
+65.96147347076715
+62.53968253968254
+70.85075797
+75.82476651212792
+75.82476651212792
+75.82476651212792
+75.82476651212792
+61.01271098145447
+66.4716169
+71.06872739230516
+74.37487224909724
+67.92328042328042
+74.95237469618341
+72.18300298669563
+61.67797407
+57.03172978505629
+47.64321004139404
+52.410732297760745
+55.00470240494424
+45.57073302677847
+50.51017005668555
+49.37316617764737
+50.675030683212874
+55.82980412916887
+62.91591046581972
+65.16554526680434
+63.45539280958722
+64.66978954
+65.72245676154161
+49.08534512070724
+57.38261738261738
+43.85296119809394
+37.76459484966968
+36.34749441010128
+38.388144539179855
+37.12136613132751
+35.378502155172406
+35.00366007852532
+37.402201299562385
+46.02073794775114
+38.523047977422394
+35.73678214836159
+34.52746958962706
+40.01340482573727
+40.01340482573727
+48.332097850259444
+32.49780420241876
+41.683353357362165
+38.63406066337904
+31.79823080559119
+32.64110756123536
+32.84710605553697
+33.22717622080679
+48.25025092004015
+52.49618776105549
+52.520390426527605
+51.84321045263649
+47.69106130164731
+30.581325502235867
+46.04050326313665
+40.12942824738141
+33.27097808658471
+38.26409102937285
+30.168050212593638
+34.95178140905438
+29.530156165858912
+34.47103526431571
+42.25286746668435
+34.89959839
+42.64420827484794
+37.5369723
+34.202350398487106
+22.01544248663631
+21.64119601328904
+37.78460306
+24.66724633803759
+16.08214428590479
+19.19456066945607
+20.34626225
+15.68251026082351
+15.28283162421436
+22.38408833898756
+17.43725231175694
+24.181327007972587
+31.552811703127098
+34.41523605150215
+42.82174903
+46.92859992022337
+30.943143812709028
+34.20895125357452
+30.98106712564544
+26.41030776420322
+22.54758418740849
+19.83668591913961
+24.16874753581285
+18.544679149730133
+21.00946792905721
+16.502068597357532
+29.654945777193557
+23.75831709120237
+22.620736237757512
+25.14499033397773
+19.59436695640455
+21.48571428571429
+25.715437323659813
+33.12583668005355
+34.213354830041645
+30.53005428
+34.10346894
+31.09371891
+30.03899422
+26.81043663471778
+33.1
+38.53583799626965
+47.30764066066873
+37.18378342860898
+32.62343354105321
+21.64294867471505
+24.68857672939306
+32.15514958962139
+21.73032797948441
+17.54432742301027
+19.937487530757462
+18.685558438102838
+23.89202256244964
+16.96505823627288
+30.68592057761733
+28.82632831086439
+21.27297481279782
+23.72609335294904
+25.92442645074224
+19.42607775720709
+36.11037934668072
+37.563762497449495
+28.06356046323727
+36.30980105130082
+21.41451164053485
+32.94615281321337
+34.68578603052931
+23.63270777479893
+24.82498653742596
+25.18718381112985
+23.94691667767067
+33.492533297457285
+27.083615666079407
+24.406104844061048
+28.42714785887423
+40.49067583652956
+37.081291535208436
+36.97541259683395
+37.26527015117619
+42.41773002014775
+37.55874760045012
+44.679128146298666
+33.53389490573606
+33.63974315647178
+36.69486445
+34.658900716341975
+43.59774716699464
+46.41222401723209
+42.46117554550816
+36.76260461318636
+36.86672904691886
+36.73578954432588
+40.95726951536564
+40.95726951536564
+50.124302895921524
+44.02350081037277
+44.02350081037277
+35.30753236300221
+34.48160085952189
+35.86591428187559
+32.56941676630796
+46.55104454079622
+43.50271002710027
+42.11298606016141
+39.53550453817405
+27.62071049627292
+27.626847782660814
+41.736060222503504
+27.42011436
+24.03318614
+26.37039020145907
+19.51170835550825
+32.26495726495726
+45.62019613040021
+45.82550336
+46.435039238044126
+40.29132700788454
+43.48115595
+46.15801599132674
+36.08828207847296
+31.04425256341069
+19.106234947819107
+32.46021220159151
+37.30815428
+21.8247347
+19.88784298017224
+20.71327825911747
+19.96553095585311
+27.48933377092222
+27.48933377092222
+23.158888589155648
+19.02820698270761
+20.93833780160858
+22.31525784157363
+30.684840425531913
+31.03610253657227
+25.57229115519873
+21.667907669396868
+24.299873611388282
+32.16358663019548
+34.69182895001004
+36.752941176470586
+36.34086909435495
+44.12963581690611
+33.37951052180223
+38.08656957928803
+21.52108934
+20.79187817258883
+22.640241854215652
+23.69263607257204
+27.900478978179883
+32.11033566
+33.100171571862205
+41.07622114868492
+22.10019659684089
+37.78715424285045
+36.24941522421974
+38.941728064300065
+36.76421571895207
+37.137330754352035
+48.64082895976315
+42.86954199492589
+40.35455714189052
+21.53087735660516
+31.65064102564103
+35.50180892402519
+41.88798493915148
+31.85888012829079
+28.86281829089455
+22.70759805553706
+28.24302134646962
+31.116879659211932
+35.25405621953662
+22.74989909861429
+24.17838811
+20.956153897804338
+26.11481975967957
+23.63806719314683
+29.450474898236088
+22.20369505769359
+29.61087420042644
+44.80021346141018
+29.75741239892183
+38.96624195455143
+36.12269608502623
+45.93449458605152
+53.694288544073956
+60.55862906490353
+48.44778302200359
+48.44778302200359
+47.71081134959133
+46.96767298958055
+50.46380933035412
+43.58356750970159
+40.521216171065824
+40.58613967869866
+48.62531545
+39.49875025332703
+40.03760660801827
+37.31465664502904
+39.10178836
+43.27321309285237
+49.01735665563585
+37.69933215210577
+35.48408755203438
+33.73799359658485
+44.42885771543086
+41.756558796111335
+35.614487249050455
+40.735068912710574
+40.873475505893566
+33.54145686
+45.12333712146534
+50.35152326749247
+40.92591346477357
+34.74265929822225
+38.27242968802191
+34.86592588
+34.24593031077627
+35.58582740571735
+40.212364015373595
+50.04080522306855
+44.33943427620632
+50.74138392
+46.56885628542848
+52.78113663845224
+61.39823717948718
+49.63430181842582
+51.47440591093656
+59.19329491880566
+54.388006571741506
+47.85757392878696
+50.75247524752476
+50.75247524752476
+48.72585285655569
+51.160477453580896
+54.49190982330242
+42.93542849597253
+46.17875211
+48.47961758066658
+61.065301236866596
+62.36602903
+34.34479054779807
+37.05664373
+49.913113220157726
+39.525850522368074
+33.547394474546785
+33.547394474546785
+33.547394474546785
+33.547394474546785
+33.547394474546785
+33.547394474546785
+37.13522537562604
+40.76119905692152
+37.64059989287627
+45.04137746930059
+48.27310840487123
+47.15893862235326
+46.54344537262382
+47.30805559303738
+50.17761989342806
+41.154799383749754
+41.18358299
+47.748830995323985
+46.61136013915836
+51.371588047103664
+39.97438835344072
+37.24435590969456
+35.52126109418466
+36.50857449
+41.46140085317187
+36.04321404456448
+35.15477792732167
+30.060862662079916
+30.060862662079916
+45.56038808966209
+45.25203580321691
+37.84554778238131
+37.21323678689856
+35.44177375324589
+38.18365287588295
+41.07466291004226
+51.06761565836299
+49.31409343859195
+50.37751677852349
+35.39974056120707
+34.317691477885646
+41.96368071554411
+34.06145624582498
+36.734141222666025
+39.29097976142608
+36.12494032599059
+37.56123750083887
+27.06761853448276
+27.06761853448276
+27.06761853448276
+27.06761853448276
+21.83423640377283
+26.22917650232684
+26.22917650232684
+26.22917650232684
+26.22917650232684
+26.22917650232684
+28.08988764044944
+25.17280719414804
+29.29394426580922
+29.29394426580922
+29.29394426580922
+37.94034282629235
+40.82730230396388
+50.739506995336434
+50.739506995336434
+41.848558340070056
+37.70755222281735
+37.70755222281735
+36.67318454588362
+41.06218487394958
+32.75277853076715
+23.62763656030287
+25.15811197656614
+25.079322216971576
+31.760058914105908
+30.24331389503318
+28.145074946466806
+23.16496972116856
+37.03877633167852
+29.91667231217397
+20.09816927566994
+23.00074109007613
+21.71229200214707
+20.38245961094626
+28.335543766578247
+26.51993021070997
+44.46682782702122
+27.698053862787496
+39.46630174478276
+30.25589225589226
+30.25589225589226
+30.25589225589226
+30.25589225589226
+22.79179280188362
+28.73057167633914
+23.477794895840308
+24.90047770700637
+25.14662757
+32.222295996281794
+39.068581398469995
+35.86261980830671
+36.53324512074099
+32.882547839610524
+37.437994369218394
+43.20592355413248
+54.086991650956094
+51.51696606786427
+51.51048295264251
+54.25772921108742
+53.499565478975875
+62.24344645397277
+44.12465128937878
+33.990246509452874
+34.27255124524271
+34.80278422273782
+31.21281773931855
+43.50019917673616
+41.73836461
+35.39929281
+35.45784591
+36.53846153846154
+41.84114324
+48.03478729859098
+45.30416221985059
+34.44762602383965
+48.86685552407932
+55.75308968343137
+49.67188964778358
+45.71600965406275
+41.10828196
+29.64016404286281
+18.85950303111052
+33.79837681937085
+23.17180022915684
+24.37945603379984
+25.15188335358445
+32.03040904198062
+29.08440439909898
+34.53328873
+29.54484244546189
+22.914072229140718
+23.79799609979154
+34.99865519096288
+25.226908702616118
+28.30554997656825
+21.81636726546906
+23.64721485411141
+18.57865318072931
+26.391683433936958
+20.440463362068968
+36.27714716882094
+51.304522344126305
+37.00536886060847
+38.71788009
+34.907625125965744
+37.36131533217484
+40.61413293091005
+40.61413293091005
+48.82452004041765
+30.761000401230444
+36.03982892274793
+30.41024960169942
+35.97184407995219
+35.97184407995219
+35.97184407995219
+23.92756915675531
+24.50066577896138
+20.61654439
+39.50192078420983
+36.67909950712668
+26.947030067635442
+19.73001728953318
+22.29939312
+21.84845850331037
+23.052035589107582
+30.15218981
+24.53214262262788
+20.23156203988757
+20.20370124
+16.834791903266748
+21.20909151503033
+26.71735360810991
+28.82678583338891
+21.99747994
+23.97219463753724
+24.52326976930257
+25.385433280170123
+25.61033021723048
+31.364855143548493
+39.799172264061326
+36.87203157401833
+36.87203157401833
+34.09618573797678
+34.55490315178511
+33.74734325185972
+23.32827249323388
+32.22018598
+23.272849462365592
+36.64471031052561
+20.74148972025615
+26.15291262135922
+34.33246161067525
+24.98662744049211
+26.15996762881036
+28.35699246077566
+28.58
+33.60879706764412
+26.4063921
+23.83155317
+17.12754697009791
+20.84893048128342
+20.84893048128342
+21.07836570663094
+22.86689419795222
+33.19057815845824
+25.73719255333605
+26.84594953519256
+37.62641484160472
+27.00845665961945
+31.21622523183668
+37.51762099751628
+38.907096515283264
+45.598344790762866
+46.69629826190792
+46.49321266968326
+46.49321266968326
+46.49321266968326
+46.49321266968326
+46.49321266968326
+43.492829379439755
+41.183542612877375
+37.50164279
+37.50164279
+47.99196787148594
+33.43110459302713
+34.34837175506969
+36.33144002672903
+47.604750467040304
+37.50666311
+34.27011188066063
+41.54155495978552
+33.852191844608534
+49.1098021
+53.95345717926363
+48.397671773600045
+47.37299465240642
+44.12955465587045
+57.13812301
+67.62585169015016
+59.44796868673235
+40.73138298
+41.66441136671177
+37.273274881199384
+33.20734699290498
+48.719839142091146
+44.573485465905996
+37.145348447246626
+37.145348447246626
+39.09603004291845
+33.76926669780476
+34.34640304588872
+31.36684944176521
+43.69592497193052
+33.23321318916033
+35.0016818
+37.15608643
+35.78169339404738
+36.05909988425138
+27.745435300344013
+33.91350561349897
+25.56527590847914
+22.06851119894598
+32.46883795737837
+23.419078242229368
+22.50731188513693
+29.44058258854684
+23.84293617595466
+19.11337209302326
+20.55042185616714
+24.87432133521013
+29.035052645608424
+29.728461280590007
+25.20766345123258
+21.61892736093573
+24.079113293953668
+21.14620513
+31.85628743
+37.537913754450756
+42.11126770015435
+33.67374182352156
+33.67374182352156
+37.66741811430845
+23.41130340724716
+19.027694759912333
+28.993101618466437
+27.75838926174497
+23.74900186318871
+21.69978225367447
+24.94662396583934
+24.94662396583934
+36.98280494357872
+31.81575789894681
+28.52637229902027
+27.707290309306632
+24.10086533261222
+25.65226184858087
+22.52739997359039
+20.03610108303249
+20.03610108303249
+26.08551321205683
+28.000000000000004
+26.997482443354983
+23.66600133067199
+22.11399237
+23.74423115510668
+22.41038482
+27.84608227553665
+38.91718038059501
+36.082952362022134
+37.05246187949217
+48.16358943815763
+30.276770119575442
+23.759599332220372
+24.93812295136798
+20.37892481756712
+22.14729251518996
+17.465142404017712
+21.249748305255387
+21.05546943
+28.622445605449382
+28.622445605449382
+28.622445605449382
+28.622445605449382
+30.01867414
+25.29131355932203
+33.00053248136315
+33.00053248136315
+33.00053248136315
+34.0544172
+35.32886440565474
+46.74579887970125
+46.74579887970125
+37.63644992880873
+45.41622055674518
+47.09146830703397
+47.06632653061224
+44.71411749313097
+48.04454101032048
+54.91059514278089
+51.362314921235885
+51.35135135135135
+55.179363160016116
+54.49288847534228
+60.16622744780054
+49.08198264846325
+49.80229207157697
+55.78926546221812
+39.156297005332966
+46.15692287180855
+42.26474193109083
+33.68818218568926
+28.69138788123069
+23.47935917037825
+22.73478289604124
+35.49820883640706
+26.239710901425422
+23.912749864937872
+25.55398396982555
+22.43662252977577
+31.48160468365378
+38.16589800147049
+29.514706872181463
+19.11676145868183
+22.63074064242102
+28.36959405677899
+41.92794547224927
+45.95495679390352
+51.14988634844231
+49.68612261252838
+52.24127656756936
+46.630136986301366
+40.79239943
+40.15100444923824
+49.00419997
+41.908319185059426
+37.429537767756486
+35.46679176998861
+35.28391272436111
+19.677721315859863
+28.737630382455198
+23.38656460340504
+23.78913567032379
+23.78913567032379
+23.32930757
+23.38806567881458
+28.72103236
+28.72103236
+35.47789250728091
+37.44640943193998
+37.44640943193998
+26.11341447104454
+26.34963161
+26.93336901257693
+30.25520483546004
+38.74017845252364
+26.670733360444633
+24.46628482166103
+20.51647978253483
+20.51647978253483
+24.41852625901706
+25.232022434399408
+30.17258612647431
+37.34061930783242
+37.16908341093521
+40.648562087142196
+40.648562087142196
+40.32301480484522
+41.13197352424693
+41.13197352424693
+41.43208890663472
+53.229368850813195
+40.86282440175261
+37.82038345105954
+33.68470525043642
+35.68561872909699
+31.50345197399289
+23.40552750465054
+24.82827609203068
+22.20369505769359
+20.09752240379547
+22.33591383372602
+22.156563309209222
+20.70950469
+28.069474945098822
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+33.61128059114636
+24.77495633481123
+23.12360914424439
+23.58610534770096
+27.72038105460888
+21.34983431392439
+40.52843117875733
+40.52843117875733
+40.52843117875733
+40.52843117875733
+39.60422878828951
+33.06419324703056
+33.06419324703056
+33.06419324703056
+33.06419324703056
+33.06419324703056
+33.06419324703056
+33.06419324703056
+36.37520938023451
+39.80634749865519
+26.793163773381778
+23.98231239432956
+35.91699306
+34.21494261
+28.714285714285708
+26.731811793052668
+23.59369761984579
+25.014989008060763
+20.14671557185729
+27.44639637
+26.28563815614175
+29.56265487911058
+34.80310777608075
+36.11504007543612
+36.89977005
+47.847682119205295
+45.62344892346905
+38.532725321888414
+38.06780115
+40.823202959830866
+55.47347213
+47.55418372139838
+27.05171255876427
+22.85905143620574
+20.84302905391929
+20.84302905391929
+19.600185344542272
+31.32441471571906
+23.94479973073039
+23.94479973073039
+23.94479973073039
+24.61435726210351
+25.09385894341647
+21.990049751243777
+23.21092278719397
+36.9600107
+36.9600107
+36.9600107
+39.02178270747027
+38.05581519562024
+42.37142668515218
+35.00575529825987
+31.41241416453481
+30.44878245119742
+19.513665594855308
+34.72112080359503
+35.33150464976249
+45.87750692146668
+45.30386740331492
+39.831977597012944
+42.00487012987013
+43.51216261246251
+36.43936849177024
+48.03569050472766
+45.58485463150778
+41.24706671
+36.247478143913916
+24.23520063567739
+23.44757380386834
+23.44757380386834
+23.44757380386834
+23.44757380386834
+23.44757380386834
+23.44757380386834
+23.44757380386834
+28.897262990876637
+30.635531378532573
+23.4909188
+23.4909188
+20.08805362071231
+26.406752411575564
+29.591291572058232
+32.55166932
+37.45939361126151
+36.35939789529772
+39.88745226770282
+51.56072455049796
+42.046839041559494
+46.68584041201257
+41.74938378522417
+47.33672376873662
+44.67803781568161
+57.07716913234706
+59.76522971
+46.713193752077096
+47.55790866975513
+58.25515947467167
+57.77372015959965
+49.12421446717476
+49.02170999731975
+54.46246798
+64.82385186175546
+61.304259771822046
+61.304259771822046
+66.18969478175254
+65.27675028582958
+64.56973293768546
+64.56973293768546
+63.322589309696475
+58.85733377881724
+58.85733377881724
+58.85733377881724
+58.85733377881724
+58.85733377881724
+58.85733377881724
+58.85733377881724
+58.85733377881724
+50.13181910363011
+43.99569921376252
+54.24008447172177
+54.24008447172177
+61.884096675123516
+38.58840811965812
+35.23051815585475
+32.13546189299312
+36.61829227134661
+44.67335681531202
+34.57708139689284
+44.97347391041569
+37.60492824262118
+35.749500333111264
+44.14601473543202
+45.186179134518625
+48.78568101670104
+50.70628997867804
+61.581771212823746
+59.85074626865672
+62.61484566065173
+70.34377285723785
+69.57506327427734
+67.27067569361476
+76.32601125100456
+62.16634169300074
+58.712811171827575
+50.82794830371567
+57.886656431479864
+58.43704001070521
+52.01934209630185
+46.38292176939035
+49.28780617678381
+45.35405640785154
+51.09015104914223
+51.78618682191056
+81.1194324
+72.35777930089102
+59.33697020562316
+50.81464312495739
+56.11660943917054
+55.68878593813871
+61.48067203933889
+60.92131809011432
+63.42780026990553
+63.41016949152542
+63.43863996
+74.05076006160853
+65.29582015211685
+53.41301644327759
+61.92267088941673
+62.33405157433795
+53.34530138783401
+54.57647942119992
+55.20920502092051
+52.59828594749014
+54.50897714907508
+54.50897714907508
+64.87209382
+53.94872499828167
+53.48305540613233
+49.92423198787712
+40.12120377384478
+46.18742327104079
+49.10562470244168
+55.785907859078584
+51.326405032134545
+63.02261593181666
+53.66174738446615
+64.32066492392251
+68.19430485762143
+67.34192756292204
+63.751838481080355
+66.22268470343393
+66.18932365201022
+73.57311941
+58.46394984326019
+39.74393713588945
+36.77419355
+50.28364431686146
+56.327576902320565
+42.26144971853499
+39.67586627788703
+42.323848238482384
+45.78583061889251
+57.588231324650785
+50.281322903801296
+42.77694098315761
+41.42138278072655
+38.26305425409113
+35.5470327
+49.56671251719395
+40.606887146266615
+48.10759664774263
+42.363250496337365
+38.74253123302553
+38.99695019
+53.43911740670117
+42.70854578829288
+42.84836485
+47.56778577676108
+41.66378535370998
+43.19457757086129
+39.83377255219947
+59.01639344262295
+53.78100423472474
+53.46289022121149
+49.25801819052178
+51.512497473556564
+53.89274534
+45.43216630196937
+41.96645929131728
+39.87372177613067
+49.48648648648649
+45.04480043442846
+51.86481755756025
+51.256281407035175
+40.31929347826087
+40.01643385
+40.47361299052774
+47.98993265764234
+38.71961656966792
+43.03880440482433
+42.47935032
+40.69539335723466
+38.03301237964237
+51.77975907
+46.25137061403509
+41.70287887017925
+41.074907625125974
+41.074907625125974
+41.074907625125974
+41.074907625125974
+41.074907625125974
+41.074907625125974
+42.02587698055767
+50.84768659236744
+60.49222797927462
+51.51098901098901
+50.609088420476326
+62.25015041112374
+65.75233022636485
+70.9166895
+58.739700121572334
+62.49496441520075
+66.15710093496212
+81.02547138604037
+74.96954108569108
+59.53737149208113
+53.77537212449256
+57.54240221529942
+63.614196694662695
+62.45254880694143
+61.88061921178936
+51.548842738270054
+51.60988017939621
+50.813559322033896
+55.080102642346894
+50.16990621
+51.93004030880645
+62.365954002832666
+52.01457645764576
+38.87735326688815
+41.20677966101695
+40.04451639012546
+49.41370088459165
+48.685384773803804
+46.665327440739254
+51.853621902095995
+54.2358918
+54.68419976834503
+51.69166610614112
+54.86471713582945
+64.87664253150979
+53.63529655458124
+59.77260422306443
+65.73071266212061
+56.08465608465608
+54.156688333903524
+56.50861181637274
+52.87118122869175
+52.36690450054885
+44.4497575
+42.982212340188994
+38.92770668
+47.752999389954596
+37.27835051546392
+40.89123662467832
+48.89466840052016
+42.46866223713953
+41.227111232577215
+43.04146829503898
+44.714660771037714
+46.59452147446737
+55.68752545133704
+65.1439853
+60.28973452554239
+51.99727335
+56.08472839084387
+45.61985756758625
+38.894814024796695
+34.26250946383096
+45.782806042951016
+49.68319559
+43.08763473254335
+61.00159092481151
+53.80707221243733
+53.859407305306696
+56.32349907159067
+53.62827083895333
+53.62827083895333
+53.62827083895333
+53.62827083895333
+53.62827083895333
+53.62827083895333
+53.62827083895333
+53.62827083895333
+53.62827083895333
+53.62827083895333
+53.62827083895333
+56.922865392547784
+64.91761210156672
+56.29014241831891
+41.53697837763393
+41.69405642289784
+35.80280914348664
+42.14960953555282
+35.398109743659475
+44.53917986195696
+52.39309928173458
+45.37690570299266
+42.964875331204574
+47.36258404380675
+47.66540966746658
+39.81690237070438
+53.80306405
+64.72717339343032
+68.72702974
+68.72702974
+68.72702974
+65.25898592319257
+66.21255299923541
+64.37992929018222
+56.31731565865783
+48.830529339351656
+41.35996716377069
+41.35996716377069
+50.54347826086957
+47.310708584912206
+38.87452368
+40.13936810770584
+41.61769789146215
+41.16554286500619
+49.28971451987433
+55.455229495861545
+39.60423634336678
+47.72788203753351
+43.42643287823134
+34.35712339961863
+48.32936372915958
+50.472885622916245
+35.47820209530247
+35.75873623743418
+41.20282378495791
+39.804197679876616
+52.99551879
+53.39786110735075
+51.05638633557415
+53.689060028879865
+53.85967284034068
+58.394561560118106
+59.828708461801995
+51.39893544424729
+51.39893544424729
+45.53323865557584
+48.90725310749898
+52.056504599211564
+65.63727855518026
+63.801272505753346
+62.85007686651962
+71.04996281522547
+60.438269044656444
+61.71128750168169
+68.48133674706914
+59.24620390455531
+50.960289795639405
+55.67663284070014
+50.62673622874178
+63.403060883512666
+61.579605445957206
+49.53894852738783
+50.87291920422249
+47.21300373
+49.79145299145299
+52.593340060544904
+49.96314907872697
+44.515993560579545
+38.40141371576157
+36.49591389
+38.75926434
+39.88415948275862
+51.86679431
+40.918977705274614
+24.58960328317373
+20.34907048281396
+31.60132262635805
+38.32652240905741
+34.155481365401656
+40.39788331435461
+56.25756150020164
+31.715498357064618
+25.81902742630463
+29.61470146194835
+26.69322709163347
+24.27471469819882
+35.96301795
+41.15160056372056
+46.73377156943303
+37.84205880355809
+36.32521634131616
+18.15708942536552
+31.89167340339531
+22.87621359223301
+22.19893163837988
+22.678258525407742
+31.97154471544715
+42.91037192602127
+28.44763387719179
+24.66711499663753
+25.33217017849953
+31.56069364
+46.71016483516484
+22.42889145
+24.47305997966791
+17.87259535379085
+18.42991913746631
+29.937921367064952
+37.34414801
+36.30748573346761
+37.241613012538124
+39.36148579949841
+36.137602179836506
+31.339031339031344
+23.07482436
+35.32351569267675
+31.182721501518078
+30.03866766162404
+20.89913312277401
+24.61486486486486
+20.149710785981632
+21.89581667998934
+19.33634585553382
+31.235258440595732
+28.635316438541807
+24.64470713
+32.42800601
+38.10509447819991
+38.10509447819991
+26.24113475177305
+26.72898462372147
+27.463978894676323
+39.02012248468941
+26.025041276829942
+26.138653768641678
+20.47122181
+22.27153520274359
+23.033098209441132
+20.19515477792732
+32.507910304030815
+38.14316182351373
+35.62075821
+38.49431818181818
+38.92130258
+35.57953311617807
+27.09530811388852
+35.065820885342056
+35.065820885342056
+33.46801346801347
+23.285723955865432
+25.51906635907179
+25.51906635907179
+25.51906635907179
+21.75080558539205
+16.20242969
+27.495786990225817
+22.99084435401831
+18.29080436675373
+22.25325884543762
+19.27346115035318
+32.84998329435349
+35.021293585307426
+37.44627388745619
+41.80183833468505
+33.90517986568256
+32.64873459652842
+34.23737916219119
+33.82815112002675
+35.24970194727779
+30.88658414185346
+45.043291496073564
+51.91087014017456
+45.17526322804212
+45.81859440699459
+47.07961061474863
+47.76922055503917
+56.40374331550802
+56.03981706
+45.63879598662207
+49.19150073499933
+47.44661095636026
+47.97292724196277
+38.47446237
+35.97208374875374
+38.50517183850517
+42.92465388711395
+39.88315089651467
+36.56254207620843
+36.56254207620843
+36.56254207620843
+36.56254207620843
+30.14442824963562
+39.00574907000338
+39.00574907000338
+39.00574907000338
+39.00574907000338
+38.518863530507694
+31.89758634484598
+22.48232626383887
+22.48232626383887
+22.48232626383887
+30.05449953476007
+32.29293338738008
+22.66933600266934
+19.50146627565982
+22.96226668457097
+15.77829405907733
+24.09407088439881
+31.33435277
+31.33435277
+31.33435277
+31.33435277
+31.33435277
+38.18291781466299
+32.10172425271287
+31.229370158302462
+32.03303343628307
+20.36877932720904
+27.60002671832209
+29.208884117291824
+28.27498493875092
+28.27498493875092
+28.27498493875092
+28.27498493875092
+24.31223290598291
+26.731133351365973
+20.74818611462424
+20.74818611462424
+30.72369296261807
+18.69891906142895
+18.69891906142895
+15.89952569
+21.89169089311068
+14.39248627554732
+25.645086781495408
+31.27770793356713
+18.038611073870488
+20.64546118600173
+18.00520729020629
+27.62716413904174
+23.80474757606152
+28.192867105001003
+21.72571121846484
+16.77972865123703
+26.393354769560563
+41.06382978723404
+31.22025383395029
+34.79713922866119
+21.36264028
+19.795768537676032
+27.922897972748416
+20.09895694035838
+34.692136011553096
+29.003800217155266
+24.76381909547739
+34.34516362678607
+38.059901056290954
+34.57317073170732
+35.12385568120625
+32.436213443474784
+26.47398843930636
+28.01961574633884
+21.724941724941722
+20.97009966777409
+21.66835528537656
+34.624907983671285
+32.32847541
+24.01372212692967
+20.51264846
+22.819283526434937
+23.81901534781742
+37.191018722100075
+36.515957446808514
+23.21085733200927
+23.21085733200927
+23.21085733200927
+21.602393617021278
+26.68438538
+22.07543430873902
+16.93769658033862
+18.21992271923505
+30.29649947753396
+19.679913196799127
+21.61732565070534
+21.61732565070534
+21.27602953895283
+21.63648558014755
+24.34577867473932
+16.91942502967163
+39.462151394422314
+24.85868102288022
+22.57001647446458
+18.96175946106598
+20.47909816815406
+19.03401541323412
+23.94974740760436
+36.95998936594444
+35.21672343551726
+34.187579045463615
+32.12974704665287
+27.31086841
+38.35762294536501
+29.30304456
+19.42186680650236
+21.65031581776643
+18.45777395674889
+20.12511646479436
+25.592766242464844
+28.40485074626866
+18.89125232650891
+24.686847599164928
+20.21905657841688
+28.15179105469529
+22.88437017649822
+27.38215318404949
+20.26686139139671
+32.91257597007948
+30.89622641509434
+23.608611611011128
+22.49634259874983
+22.58958622055039
+33.69702614596501
+33.69702614596501
+32.09868244151654
+35.666912306558594
+32.77645186953063
+32.04888473353813
+32.912730395719095
+40.16941301038978
+38.41577540106952
+36.18870678787075
+38.76273556635813
+55.75916230366492
+50.75052639
+48.83176005891411
+49.75300400534045
+58.64484407624436
+54.049001936043794
+64.36445743126576
+46.32503062474479
+56.23274161735699
+53.762079306897704
+46.85549365729244
+44.75849973
+33.08661732346469
+33.08661732346469
+33.08661732346469
+38.01349275265513
+36.14620633740963
+36.14620633740963
+36.14620633740963
+42.91154462854089
+44.07005393886928
+37.81862087469815
+37.81862087469815
+37.81862087469815
+33.980839114634946
+36.30320934604539
+51.25967720771551
+36.73871669385536
+41.08848916123081
+37.84180338802188
+35.68517404957573
+31.49969885565148
+35.84502338009352
+33.00080342795929
+38.48350043157825
+38.32463535394085
+40.38901601830664
+40.38901601830664
+40.38901601830664
+40.38901601830664
+40.38901601830664
+40.38901601830664
+48.87621375704488
+52.569169960474305
+47.09051007514797
+58.33610207
+53.54277165782674
+40.26438254535644
+36.93499494779387
+32.44394171
+32.44394171
+32.44394171
+32.44394171
+46.01247568582735
+40.06772773450728
+30.052712350703942
+30.052712350703942
+35.18879989230666
+38.243739565943244
+44.189033871949164
+34.81046096948884
+47.20442436096311
+42.51197948302625
+38.54737404784177
+37.91305525460455
+36.10758011641132
+37.039045553145336
+43.15297261189045
+38.853503184713375
+35.17715353306199
+34.89333333333333
+36.4681366
+47.789981248325745
+50.704129532621266
+48.58399413606983
+50.54161340240867
+42.66765920010828
+47.63835394283398
+55.947756195579366
+40.381515314347126
+37.71941668917094
+36.63983903420523
+39.27550681289465
+44.166045913184796
+42.391963864356505
+30.46038543897216
+35.610406937958636
+26.141911322142708
+20.298746332355293
+31.26879134095009
+28.251121076233183
+25.16595857673925
+20.47244094488189
+23.87831180934629
+37.09839694144476
+35.42522389064709
+24.06247896047936
+28.891257995735607
+26.25462229265716
+31.15516892575359
+22.915549597855232
+18.08574677279112
+20.33028488716363
+24.083769633507853
+22.035124646451358
+26.93534161069272
+34.10205434062293
+36.0267798
+37.23931798111541
+34.47232125
+35.82802547770701
+41.32608549761761
+36.17207486199004
+32.578733604101465
+32.19418504536724
+32.19418504536724
+32.19418504536724
+32.19418504536724
+32.19418504536724
+39.36690457309014
+44.14117647058824
+41.21171770972037
+38.58634217700645
+34.13351314902225
+38.34731655346917
+46.93023883416543
+46.23663101604278
+40.03473149879776
+38.6596548
+37.68770764119601
+36.97176046465051
+38.74873182279337
+40.929888305746196
+20.40816326530612
+22.49581799933088
+30.79198159710812
+27.20874111811235
+23.46306068601583
+32.06847745501112
+25.13432053466125
+25.58517115410635
+28.49224678323985
+28.404433993953642
+26.68164398044139
+36.97305229
+35.038873014818265
+33.93955783922656
+34.64409490802453
+34.41190206151178
+27.013772253946932
+26.579765764573548
+19.936457505957108
+23.63252933
+22.977368207074132
+35.26058085720035
+23.64499538
+23.04955015
+23.04955015
+23.04955015
+23.04955015
+23.04955015
+23.04955015
+23.04955015
+23.34434897554527
+22.448707700506258
+22.448707700506258
+22.448707700506258
+22.448707700506258
+22.448707700506258
+28.209897044171374
+22.66862758205762
+30.29136523528241
+23.64253393665158
+35.020730239400834
+15.26242637
+15.26242637
+19.38809444629199
+20.70224346210009
+30.03630984400215
+27.27943148297131
+26.20588431516445
+26.779999999999998
+40.23302263648469
+35.55763573147901
+34.06512745
+36.28096995255667
+36.00943714189417
+29.27350427350427
+30.434201736806948
+19.78631051752922
+20.561630889698932
+19.86618972
+34.606096207819334
+25.517897167696802
+21.29851143009038
+23.62911266201396
+27.19222318396698
+42.415504291845494
+36.84175038770144
+35.95824800158552
+35.313066237165295
+35.555555555555564
+40.91728419787703
+21.94044831047173
+15.680551870522692
+15.680551870522692
+15.680551870522692
+15.680551870522692
+15.680551870522692
+15.680551870522692
+15.680551870522692
+18.95781968
+20.920225091029458
+18.104076897580377
+18.104076897580377
+18.104076897580377
+18.104076897580377
+18.104076897580377
+18.104076897580377
+18.104076897580377
+18.104076897580377
+28.48270335247763
+21.39065817409766
+21.39065817409766
+22.15596944928313
+27.389292795769993
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+33.09257173149559
+35.466191230850505
+35.38969795037756
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+34.33124416588879
+32.78601342817257
+46.3142418
+46.3142418
+46.3142418
+46.3142418
+22.76794725156429
+17.417978127500668
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
+17.777480967009478
diff --git a/atsc/tests/e2e.rs b/atsc/tests/e2e.rs
new file mode 100644
index 0000000..62a2d63
--- /dev/null
+++ b/atsc/tests/e2e.rs
@@ -0,0 +1,271 @@
+use atsc::csv::{read_samples, read_samples_with_headers};
+use atsc::utils::error::calculate_error;
+use std::fs;
+use std::path::{Path, PathBuf};
+use wavbrro::wavbrro::WavBrro;
+
+const TEST_FILE_NAME: &str = "go_gc_heap_goal_bytes.wbro";
+const TEST_COMPRESSED_FILE_NAME: &str = "go_gc_heap_goal_bytes.bro";
+const TEST_WBRO_PATH: &str = "tests/wbros/go_gc_heap_goal_bytes.wbro";
+
+#[test]
+fn test_compressor_idw_lossless() {
+ test_lossless_compression("idw")
+}
+
+#[test]
+fn test_compressor_idw_lossy() {
+ test_lossy_compression("idw")
+}
+
+#[test]
+fn test_compressor_polynomial_lossless() {
+ test_lossless_compression("polynomial")
+}
+
+#[test]
+fn test_compressor_polynomial_lossy() {
+ test_lossy_compression("polynomial")
+}
+
+#[test]
+fn test_compressor_noop() {
+ test_lossless_compression("noop")
+}
+
+#[test]
+fn test_compressor_fft_lossy() {
+ test_lossy_compression("fft")
+}
+
+#[test]
+fn test_compressor_auto_lossless() {
+ test_lossless_compression("auto")
+}
+
+#[test]
+fn test_compressor_auto_lossy() {
+ test_lossy_compression("auto")
+}
+
+#[test]
+fn test_csv_input_compression_with_header() {
+ let filepath = Path::new("./tests/csv/cpu_utilization.csv");
+ let test_dir = prepare_test_dir_and_copy_file(filepath);
+ let csv_file_path = test_dir.join("cpu_utilization.csv");
+
+ run_compressor(&[
+ "--compressor",
+ "noop",
+ "--error",
+ "5",
+ "--csv",
+ "--fields=time,value",
+ csv_file_path.to_str().unwrap(),
+ ]);
+
+ run_compressor(&["-u", test_dir.join("cpu_utilization.bro").to_str().unwrap()]);
+
+ let uncompressed_samples = WavBrro::from_file(&test_dir.join("cpu_utilization.wbro")).unwrap();
+ let original_samples = read_values_from_csv(&csv_file_path, false);
+
+ let err = calculate_error(&original_samples, &uncompressed_samples);
+
+ assert!(
+ err <= 0.05,
+ "Error: {}\nOriginal : {:?}\nUncompressed: {:?}",
+ err,
+ original_samples,
+ uncompressed_samples
+ );
+}
+
+#[test]
+fn test_csv_input_compression_with_header_no_fields() {
+ let filepath = Path::new("./tests/csv/cpu_utilization.csv");
+ let test_dir = prepare_test_dir_and_copy_file(filepath);
+ let csv_file_path = test_dir.join("cpu_utilization.csv");
+
+ run_compressor(&[
+ "--compressor",
+ "noop",
+ "--error",
+ "5",
+ "--csv",
+ csv_file_path.to_str().unwrap(),
+ ]);
+
+ run_compressor(&["-u", test_dir.join("cpu_utilization.bro").to_str().unwrap()]);
+
+ let uncompressed_samples = WavBrro::from_file(&test_dir.join("cpu_utilization.wbro")).unwrap();
+ let original_samples = read_values_from_csv(&csv_file_path, false);
+
+ let err = calculate_error(&original_samples, &uncompressed_samples);
+
+ assert!(
+ err <= 0.05,
+ "Error: {}\nOriginal : {:?}\nUncompressed: {:?}",
+ err,
+ original_samples,
+ uncompressed_samples
+ );
+}
+
+#[test]
+fn test_csv_input_compression_without_header() {
+ let filepath = Path::new("./tests/csv/cpu_utilization_no_headers_only_values.csv");
+ let test_dir = prepare_test_dir_and_copy_file(filepath);
+ let csv_file_path = test_dir.join("cpu_utilization_no_headers_only_values.csv");
+
+ run_compressor(&[
+ "--compressor",
+ "noop",
+ "--error",
+ "5",
+ "--csv",
+ "--no-header",
+ csv_file_path.to_str().unwrap(),
+ ]);
+
+ run_compressor(&[
+ "-u",
+ test_dir
+ .join("cpu_utilization_no_headers_only_values.bro")
+ .to_str()
+ .unwrap(),
+ ]);
+
+ let uncompressed_samples =
+ WavBrro::from_file(&test_dir.join("cpu_utilization_no_headers_only_values.wbro")).unwrap();
+ let original_samples = read_values_from_csv(&csv_file_path, true);
+
+ let err = calculate_error(&original_samples, &uncompressed_samples);
+
+ assert!(
+ err <= 0.05,
+ "Error: {}\nOriginal : {:?}\nUncompressed: {:?}",
+ err,
+ original_samples,
+ uncompressed_samples
+ );
+}
+
+fn test_lossless_compression(compressor: &str) {
+ test_compression_decompression_flow(compressor, 0, compare_samples_lossless)
+}
+
+fn test_lossy_compression(compressor: &str) {
+ test_compression_decompression_flow(compressor, 5, compare_samples_with_allowed_error)
+}
+
+#[test]
+fn test_compressor_constant() {
+ // tests/wbros/uptime.wbro constant data which can be compressed by constant compressor
+ let test_dir = tempfile::tempdir().unwrap().into_path();
+ fs::copy("tests/wbros/uptime.wbro", test_dir.join("uptime.wbro")).unwrap();
+
+ run_compressor(&[
+ "--compressor",
+ "noop",
+ test_dir.join("uptime.wbro").to_str().unwrap(),
+ ]);
+
+ run_compressor(&["-u", test_dir.join("uptime.bro").to_str().unwrap()]);
+
+ compare_samples_lossless(
+ &PathBuf::from("tests/wbros/uptime.wbro"),
+ &test_dir.join("uptime.wbro"),
+ )
+}
+
+/// Runs compression and decompression test for a specified compressor.
+/// max_error is an error level, compression speed is set as the lowest (0).
+///
+/// It compresses tests/wbros/go_gc_duration_count.wbro file, decompresses
+/// got .bro file, and compares the original .wbro and the decompressed one.
+fn test_compression_decompression_flow(
+ compressor: &str,
+ allowed_error: u8,
+ compare_fn: fn(original: &Path, processed: &Path),
+) {
+ let test_dir = prepare_test_dir();
+
+ // Running data compression
+ run_compressor(&[
+ "--compressor",
+ compressor,
+ "--error",
+ allowed_error.to_string().as_str(),
+ test_dir.join(TEST_FILE_NAME).to_str().unwrap(),
+ ]);
+
+ // Running data decompression
+ run_compressor(&[
+ "-u",
+ test_dir.join(TEST_COMPRESSED_FILE_NAME).to_str().unwrap(),
+ ]);
+
+ compare_fn(
+ &PathBuf::from(TEST_WBRO_PATH),
+ &test_dir.join(TEST_FILE_NAME),
+ )
+}
+
+/// Prepares test directory and copies test wbro file there.
+fn prepare_test_dir() -> PathBuf {
+ let test_dir = tempfile::tempdir().unwrap().into_path();
+ fs::copy(TEST_WBRO_PATH, test_dir.join(TEST_FILE_NAME)).unwrap();
+ test_dir
+}
+
+/// Prepares test directory and copies file to it.
+fn prepare_test_dir_and_copy_file(filepath: &Path) -> PathBuf {
+ let test_dir = tempfile::tempdir().unwrap().into_path();
+ fs::copy(filepath, test_dir.join(filepath.file_name().unwrap())).unwrap();
+ test_dir
+}
+
+/// Runs compressor binary with provided arguments.
+fn run_compressor(args: &[&str]) {
+ let compressor_bin = env!("CARGO_BIN_EXE_atsc");
+ let exit_status = std::process::Command::new(compressor_bin)
+ .args(args)
+ .status()
+ .unwrap();
+
+ assert!(exit_status.success());
+}
+
+fn compare_samples_lossless(original: &Path, uncompressed: &Path) {
+ let original_samples = WavBrro::from_file(original).unwrap();
+ let uncompressed_samples = WavBrro::from_file(uncompressed).unwrap();
+ assert_eq!(original_samples, uncompressed_samples);
+}
+
+fn compare_samples_with_allowed_error(original: &Path, uncompressed: &Path) {
+ const MAX_ALLOWED_ERROR: f64 = 0.05;
+
+ let original_samples = WavBrro::from_file(original).unwrap();
+ let uncompressed_samples = WavBrro::from_file(uncompressed).unwrap();
+ let err = calculate_error(&original_samples, &uncompressed_samples);
+
+ assert!(
+ err <= MAX_ALLOWED_ERROR,
+ "Error: {}\nOriginal : {:?}\nUncompressed: {:?}",
+ err,
+ original_samples,
+ uncompressed_samples
+ );
+}
+
+/// Reads csv file and returns the content of value field.
+/// If no_headers = false it assumes --fields="time,value"
+fn read_values_from_csv(csv_file_path: &Path, no_headers: bool) -> Vec {
+ let samples = if no_headers {
+ read_samples(csv_file_path).unwrap()
+ } else {
+ read_samples_with_headers(csv_file_path, "time", "value").unwrap()
+ };
+
+ samples.into_iter().map(|sample| sample.value).collect()
+}
diff --git a/brro-compressor/tests/integration_test.rs b/atsc/tests/integration_test.rs
similarity index 78%
rename from brro-compressor/tests/integration_test.rs
rename to atsc/tests/integration_test.rs
index e35add1..7b1e4e7 100644
--- a/brro-compressor/tests/integration_test.rs
+++ b/atsc/tests/integration_test.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use tempfile::tempdir;
#[test]
@@ -86,7 +102,7 @@ fn compress_file_with_speed(speed: u8) {
fn run_compressor(args: &[&str]) {
// path to binary set by cargo: https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates
- let command = std::env!("CARGO_BIN_EXE_brro-compressor");
+ let command = std::env!("CARGO_BIN_EXE_atsc");
let status = std::process::Command::new(command)
.args(args)
diff --git a/atsc/tests/wbros/go_gc_heap_goal_bytes.wbro b/atsc/tests/wbros/go_gc_heap_goal_bytes.wbro
new file mode 100644
index 0000000..d6741a9
Binary files /dev/null and b/atsc/tests/wbros/go_gc_heap_goal_bytes.wbro differ
diff --git a/brro-compressor/tests/wbros/memory_used.wbro b/atsc/tests/wbros/memory_used.wbro
similarity index 100%
rename from brro-compressor/tests/wbros/memory_used.wbro
rename to atsc/tests/wbros/memory_used.wbro
diff --git a/brro-compressor/tests/wbros/uptime.wbro b/atsc/tests/wbros/uptime.wbro
similarity index 100%
rename from brro-compressor/tests/wbros/uptime.wbro
rename to atsc/tests/wbros/uptime.wbro
diff --git a/brro-compressor/src/compare.rs b/brro-compressor/src/compare.rs
deleted file mode 100644
index c34b172..0000000
--- a/brro-compressor/src/compare.rs
+++ /dev/null
@@ -1,125 +0,0 @@
-use crate::{
- compressor::{
- constant::constant_compressor,
- fft::fft,
- polynomial::{polynomial, PolynomialType},
- },
- optimizer::utils::DataStats,
-};
-use std::thread;
-
-/// Enum to represent the decision between compressors.
-#[derive(PartialEq, Debug)]
-enum CompressionDecision {
- Constant,
- Fft,
- Polynomial,
-}
-
-impl CompressionDecision {
- /// Function to perform compression and make a decision based on the results.
- pub fn compress_and_decide() -> Result<(), Box> {
- // Sample data for testing
- let data = vec![1.0, 2.0, 3.0, 4.0, 5.0];
- let stats = DataStats::new(&data);
-
- // Clone data for each compressor
- let data_constant = data.clone();
- let data_fft = data.clone();
- let data_polynomial = data.clone();
-
- // Create threads for each compressor
- let thread_constant = thread::spawn(move || constant_compressor(&data_constant, stats));
- let thread_fft = thread::spawn(move || fft(&data_fft));
- let thread_polynomial =
- thread::spawn(move || polynomial(&data_polynomial, PolynomialType::Polynomial));
-
- // Wait for threads to finish and collect their results with error handling
- let result_constant = thread_constant
- .join()
- .map_err(|e| format!("Constant thread error: {:?}", e))?;
- let result_fft = thread_fft
- .join()
- .map_err(|e| format!("FFT thread error: {:?}", e))?;
- let result_polynomial = thread_polynomial
- .join()
- .map_err(|e| format!("Polynomial thread error: {:?}", e))?;
-
- // Use the decision logic to determine the compression decision
- let decision = match (
- result_constant.compressed_data.len(),
- result_fft.len(),
- result_polynomial.len(),
- ) {
- (constant_len, fft_len, poly_len)
- if constant_len < fft_len && constant_len < poly_len =>
- {
- CompressionDecision::Constant
- }
- (_, fft_len, poly_len) if fft_len < poly_len => CompressionDecision::Fft,
- _ => CompressionDecision::Polynomial,
- };
-
- // Use the decision to perform further actions
- match decision {
- CompressionDecision::Constant => {
- println!("Selected Constant Compressor");
- }
- CompressionDecision::Fft => {
- println!("Selected FFT Compressor");
- }
- CompressionDecision::Polynomial => {
- println!("Selected Polynomial Compressor");
- }
- }
-
- Ok(())
- }
-}
-fn get_compression_decision(
- result_constant: &[f64],
- result_fft: &[f64],
- result_polynomial: &[f64],
-) -> CompressionDecision {
- match (
- result_constant.len(),
- result_fft.len(),
- result_polynomial.len(),
- ) {
- (constant_len, fft_len, poly_len) if constant_len < fft_len && constant_len < poly_len => {
- CompressionDecision::Constant
- }
- (_, fft_len, poly_len) if fft_len < poly_len => CompressionDecision::Fft,
- _ => CompressionDecision::Polynomial,
- }
-}
-#[cfg(test)]
-mod tests {
- use super::*;
- #[test]
- fn test_comparison_logic_constant_wins() {
- let result_constant = vec![1.0, 2.0, 3.0];
- let result_fft = vec![1.0, 2.0, 3.0, 4.0, 5.0];
- let result_polynomial = vec![1.0, 2.0, 3.0, 4.0, 5.0];
- let decision = get_compression_decision(&result_constant, &result_fft, &result_polynomial);
- assert_eq!(decision, CompressionDecision::Constant);
- }
-
- #[test]
- fn test_comparison_logic_fft_wins() {
- let result_constant = vec![1.0, 2.0, 3.0, 4.0, 5.0];
- let result_fft = vec![1.0, 2.0, 3.0];
- let result_polynomial = vec![1.0, 2.0, 3.0, 4.0, 5.0];
- let decision = get_compression_decision(&result_constant, &result_fft, &result_polynomial);
- assert_eq!(decision, CompressionDecision::Fft);
- }
-
- #[test]
- fn test_comparison_logic_polynomial_wins() {
- let result_constant = vec![1.0, 2.0, 3.0, 4.0, 5.0];
- let result_fft = vec![1.0, 2.0, 3.0, 4.0, 5.0];
- let result_polynomial = vec![1.0, 2.0, 3.0];
- let decision = get_compression_decision(&result_constant, &result_fft, &result_polynomial);
- assert_eq!(decision, CompressionDecision::Polynomial);
- }
-}
diff --git a/brro-compressor/src/header.rs b/brro-compressor/src/header.rs
deleted file mode 100644
index a93e7ae..0000000
--- a/brro-compressor/src/header.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-use bincode::{Decode, Encode};
-
-/// This will write the file headers
-#[derive(Encode, Decode, Debug, Clone)]
-pub struct CompressorHeader {
- initial_segment: [u8; 4],
- // We should go unsigned
- frame_count: i16,
-}
-
-impl CompressorHeader {
- pub fn new() -> Self {
- CompressorHeader {
- initial_segment: *b"BRRO",
- // We have to limit the bytes of the header
- frame_count: 0,
- }
- }
-
- pub fn add_frame(&mut self) {
- self.frame_count += 1;
- }
-}
diff --git a/brro-compressor/src/lib.rs b/brro-compressor/src/lib.rs
deleted file mode 100644
index 95fa5fd..0000000
--- a/brro-compressor/src/lib.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-#![allow(clippy::new_without_default)]
-// Lucas - Once the project is far enough along I strongly reccomend reenabling dead code checks
-#![allow(dead_code)]
-
-pub mod compare;
-pub mod compressor;
-pub mod data;
-pub mod frame;
-pub mod header;
-pub mod preprocessor;
-pub mod utils;
-
-pub mod optimizer;
-pub mod types;
diff --git a/brro-compressor/src/types/metric_tag.rs b/brro-compressor/src/types/metric_tag.rs
deleted file mode 100644
index ff89342..0000000
--- a/brro-compressor/src/types/metric_tag.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-use median::Filter;
-
-#[derive(Debug)]
-pub enum MetricTag {
- Percent(i32),
- // If it is a percent reduce significant digits to 2
- Duration(i32),
- // if it is a duration reduce precision to 1 microsecond
- NotFloat,
- // A metric that has a float representation but shouldn't (Eg. Precision is not needed)
- QuasiRandom,
- // A metric that exhibits a quasi random sample behavior. (E.g. Network deltas, heap memory)
- Bytes(i32),
- // Data that is in bytes... Make it MB, or KB
- Other, // Everything else
-}
-
-impl MetricTag {
- #[allow(clippy::wrong_self_convention)]
- fn from_float(&self, x: f64) -> i64 {
- match self {
- MetricTag::Other => 0,
- MetricTag::NotFloat | MetricTag::QuasiRandom => x as i64,
- MetricTag::Percent(y) => Self::to_multiply_and_truncate(x, *y),
- MetricTag::Duration(y) => Self::to_multiply_and_truncate(x, *y),
- MetricTag::Bytes(y) => (x as i64) / (*y as i64),
- }
- }
-
- /// Converts a float via multiplication and truncation
- fn to_multiply_and_truncate(number: f64, mul: i32) -> i64 {
- (number * mul as f64) as i64
- }
-
- fn to_median_filter(data: &[f64]) -> Vec {
- let mut filtered = Vec::with_capacity(data.len());
- // 10minutes of data
- let mut filter = Filter::new(50);
- for point in data {
- let point_int = MetricTag::QuasiRandom.from_float(*point);
- let median = filter.consume(point_int);
- filtered.push(median)
- }
- filtered
- }
-}
diff --git a/brro-compressor/src/types/mod.rs b/brro-compressor/src/types/mod.rs
deleted file mode 100644
index 88c599e..0000000
--- a/brro-compressor/src/types/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod metric_tag;
diff --git a/brro-compressor/src/utils/readers/mod.rs b/brro-compressor/src/utils/readers/mod.rs
deleted file mode 100644
index b2ab21c..0000000
--- a/brro-compressor/src/utils/readers/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod bro_reader;
diff --git a/brro-compressor/src/utils/writers/mod.rs b/brro-compressor/src/utils/writers/mod.rs
deleted file mode 100644
index 8b13789..0000000
--- a/brro-compressor/src/utils/writers/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/csv-compressor/Cargo.toml b/csv-compressor/Cargo.toml
index 65c86ca..8a30324 100644
--- a/csv-compressor/Cargo.toml
+++ b/csv-compressor/Cargo.toml
@@ -2,12 +2,12 @@
name = "csv-compressor"
version = "0.1.0"
edition = "2021"
-description = "Utilizes brro-compressor functionalities to compress CSV formatted data"
+description = "Utilizes ATSC functionalities to compress CSV formatted data"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-brro-compressor = { version = "0.5.0", path = "../brro-compressor" }
+atsc = { version = "0.5.0", path = "../atsc" }
clap = { workspace = true, features = ["derive"] }
csv = "1.3.0"
serde = { version = "1.0.171", features = ["derive"] }
diff --git a/csv-compressor/src/csv.rs b/csv-compressor/src/csv.rs
index ff6f0f4..a0fdc09 100644
--- a/csv-compressor/src/csv.rs
+++ b/csv-compressor/src/csv.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use serde::{Deserialize, Serialize};
use std::fs::{File, OpenOptions};
use std::path::Path;
@@ -73,7 +89,6 @@ mod tests {
TempDir::new("test_read_samples").expect("Unable to create temporary directory");
let path = temp_dir.path().join("samples.csv");
- // Writing content to test file
let mut file = File::create(&path).expect("Unable to create test file");
file.write_all(csv_content.as_bytes())
.expect("Unable to write data");
diff --git a/csv-compressor/src/main.rs b/csv-compressor/src/main.rs
index 032a5d4..a56dd0c 100644
--- a/csv-compressor/src/main.rs
+++ b/csv-compressor/src/main.rs
@@ -1,8 +1,24 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use crate::metric::Metric;
-use brro_compressor::compressor::Compressor;
-use brro_compressor::data::CompressedStream;
-use brro_compressor::optimizer::OptimizerPlan;
-use brro_compressor::utils::readers::bro_reader::read_file;
+use atsc::compressor::Compressor;
+use atsc::data::CompressedStream;
+use atsc::optimizer::OptimizerPlan;
+use atsc::utils::readers::bro_reader::read_file;
use clap::{arg, Parser};
use log::debug;
use std::fs;
@@ -18,7 +34,6 @@ mod metric;
author, version, about = "A Time-Series compressor utilizes Brro Compressor for CSV format", long_about = None
)]
pub struct Args {
- /// Path to input
input: PathBuf,
/// Defines where the result will be stored
@@ -76,7 +91,6 @@ enum CompressorType {
Idw,
}
-/// Compresses the data based on the provided tag and arguments.
fn compress_data(vec: &[f64], arguments: &Args) -> Vec {
debug!("Compressing data!");
//let optimizer_results = optimizer::process_data(vec, tag);
@@ -131,7 +145,6 @@ fn process_args(args: Args) {
.unwrap_or_else(|| args.input.clone())
.clone();
- // uncompressing input
if args.uncompress {
debug!("Starting uncompressing of {:?}", &args.input);
if let Some(data) = read_file(&args.input).expect("failed to read bro file") {
diff --git a/csv-compressor/src/metric.rs b/csv-compressor/src/metric.rs
index f568b56..f508d00 100644
--- a/csv-compressor/src/metric.rs
+++ b/csv-compressor/src/metric.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use crate::csv::Sample;
use std::fmt::{Debug, Display, Formatter};
use std::path::Path;
@@ -7,9 +23,7 @@ use wavbrro::wavbrro::WavBrro;
/// Metric is responsible for generating WavBrro and VSRI from parsed Samples
#[derive(Default)]
pub struct Metric {
- /// Metric data itself
pub wbro: WavBrro,
- /// Metric indexes
pub vsri: Vsri,
}
@@ -38,7 +52,6 @@ impl Metric {
Metric { wbro, vsri }
}
- /// Appends samples to the metric
pub fn append_samples(&mut self, samples: &[Sample]) -> Result<(), Error> {
for sample in samples {
// For solution simplification it generates only 1 WavBrro and 1 VSRI
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 0000000..e69de29
diff --git a/optimizer/Cargo.toml b/optimizer/Cargo.toml
deleted file mode 100644
index d3004d1..0000000
--- a/optimizer/Cargo.toml
+++ /dev/null
@@ -1,18 +0,0 @@
-[package]
-name = "optimizer"
-version = "0.1.0"
-authors = ["Carlos Rolo "]
-edition = "2021"
-license = "Apache-2.0"
-description = "Optimizer stage for the compressor."
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-[dependencies]
-hound = "3.5"
-chrono = "0.4.26"
-claxon = "0.4.3"
-env_logger = "0.11.0"
-log = "0.4.0"
-clap = {version = "4.3.14", features = ["derive"] }
-regex = "1.9.1"
-median = "0.3.2"
\ No newline at end of file
diff --git a/optimizer/src/main.rs b/optimizer/src/main.rs
deleted file mode 100644
index b6b563b..0000000
--- a/optimizer/src/main.rs
+++ /dev/null
@@ -1,438 +0,0 @@
-// Lucas - Once the project is far enough along I strongly reccomend reenabling dead code checks
-#![allow(dead_code)]
-
-use clap::{arg, command, Parser};
-use hound::{WavSpec, WavWriter};
-use log::{debug, error, info};
-use median::Filter;
-use regex::Regex;
-use std::fs;
-use std::io::Write;
-use std::path::PathBuf;
-use std::{fs::File, path::Path};
-
-#[derive(Debug)]
-enum MetricTag {
- Percent(i32), // If it is a percent reduce significant digits to 2
- Duration(i32), // if it is a duration reduce precision to 1 microsecond
- NotFloat, // A metric that has a float representation but shouldn't (Eg. Precision is not needed)
- QuasiRandom, // A metric that exhibits a quasi random sample behavior. (E.g. Network deltas, heap memory)
- Bytes(i32), // Data that is in bytes... Make it MB, or KB
- Other, // Everything else
-}
-
-impl MetricTag {
- #[allow(clippy::wrong_self_convention)]
- fn from_float(&self, x: f64) -> i64 {
- match self {
- MetricTag::Other => 0,
- MetricTag::NotFloat | MetricTag::QuasiRandom => x as i64,
- MetricTag::Percent(y) => to_multiply_and_truncate(x, *y),
- MetricTag::Duration(y) => to_multiply_and_truncate(x, *y),
- MetricTag::Bytes(y) => (x as i64) / (*y as i64),
- }
- }
-}
-
-/*
-Reads a WAV file, checks the channels and the information contained there. From that
-information takes a decision on the best channel, block size and bitrate for the BRRO
-encoders.
-*/
-
-/* Read a WAV file, */
-fn read_metrics_from_wav(filename: &str) -> Vec {
- let r_reader = hound::WavReader::open(filename);
- let mut reader = match r_reader {
- Ok(reader) => reader,
- Err(_err) => {
- return Vec::new();
- }
- };
- let num_channels = reader.spec().channels as usize;
-
- let mut raw_data: Vec = Vec::new();
- let mut u64_holder: [u16; 4] = [0, 0, 0, 0];
-
- // Iterate over the samples and channels and push each sample to the vector
- let mut current_channel: usize = 0;
- for sample in reader.samples::() {
- u64_holder[current_channel] = sample.unwrap() as u16;
- current_channel += 1;
- if current_channel == num_channels {
- raw_data.push(join_u16_into_f64(u64_holder));
- current_channel = 0;
- }
- }
- raw_data
-}
-
-fn generate_wav_header(channels: Option, bitdepth: u16, samplerate: u32) -> WavSpec {
- hound::WavSpec {
- channels: channels.unwrap_or(4) as u16,
- // TODO: Sample rate adaptations
- sample_rate: samplerate,
- bits_per_sample: bitdepth,
- sample_format: hound::SampleFormat::Int,
- }
-}
-
-/// Write a WAV file with the outputs of data analysis for float data
-fn write_optimal_wav(filename: &str, data: Vec, bitdepth: i32, dc: i64, channels: i32) {
- // Make DC a float for operations
- let fdc = dc as f64;
- let header: WavSpec = generate_wav_header(Some(channels), bitdepth as u16, 8000);
- let mut file_path = filename.to_string();
- file_path.truncate(file_path.len() - 4);
- file_path = format!("{}_OPT.wav", file_path);
- let file = File::create(file_path).unwrap();
- let mut wav_writer = WavWriter::new(file, header).unwrap();
- for sample in data {
- let _ = match bitdepth {
- 8 => wav_writer.write_sample(as_i8(sample - fdc)),
- 16 => wav_writer.write_sample(as_i16(sample - fdc)),
- _ => wav_writer.write_sample(as_i32(sample - fdc)),
- };
- }
- let _ = wav_writer.finalize();
-}
-
-fn write_optimal_int_wav(filename: &str, data: Vec, bitdepth: i32, dc: i64, channels: i32) {
- let header: WavSpec = generate_wav_header(Some(channels), bitdepth as u16, 8000);
- let mut file_path = filename.to_string();
- file_path.truncate(file_path.len() - 4);
- file_path = format!("{}_OPT.wav", file_path);
- let file = File::create(file_path).unwrap();
- let mut wav_writer = WavWriter::new(file, header).unwrap();
- for sample in data {
- let _ = match bitdepth {
- 8 => wav_writer.write_sample((sample - dc) as i8),
- 16 => wav_writer.write_sample((sample - dc) as i16),
- _ => wav_writer.write_sample((sample - dc) as i32),
- };
- }
- let _ = wav_writer.finalize();
-}
-
-fn as_i8(value: f64) -> i8 {
- split_n(value).0 as i8
-}
-
-fn as_i16(value: f64) -> i16 {
- split_n(value).0 as i16
-}
-
-fn as_i32(value: f64) -> i32 {
- split_n(value).0 as i32
-}
-
-// Split a float into an integer
-fn split_n(x: f64) -> (i64, f64) {
- const FRACT_SCALE: f64 = 1.0 / (65536.0 * 65536.0 * 65536.0 * 65536.0); // 1_f64.exp(-64)
- const STORED_MANTISSA_DIGITS: u32 = f64::MANTISSA_DIGITS - 1;
- const STORED_MANTISSA_MASK: u64 = (1 << STORED_MANTISSA_DIGITS) - 1;
- const MANTISSA_MSB: u64 = 1 << STORED_MANTISSA_DIGITS;
-
- const EXPONENT_BITS: u32 = 64 - 1 - STORED_MANTISSA_DIGITS;
- const EXPONENT_MASK: u32 = (1 << EXPONENT_BITS) - 1;
- const EXPONENT_BIAS: i32 = (1 << (EXPONENT_BITS - 1)) - 1;
-
- let bits = x.to_bits();
- let is_negative = (bits as i64) < 0;
- let exponent = ((bits >> STORED_MANTISSA_DIGITS) as u32 & EXPONENT_MASK) as i32;
-
- let mantissa = (bits & STORED_MANTISSA_MASK) | MANTISSA_MSB;
- let mantissa = if is_negative {
- -(mantissa as i64)
- } else {
- mantissa as i64
- };
-
- let shl = exponent + (64 - f64::MANTISSA_DIGITS as i32 - EXPONENT_BIAS + 1);
- if shl <= 0 {
- let shr = -shl;
- if shr < 64 {
- // x >> 0..64
- let fraction = ((mantissa as u64) >> shr) as f64 * FRACT_SCALE;
- (0, fraction)
- } else {
- // x >> 64..
- (0, 0.0)
- }
- } else if shl < 64 {
- // x << 1..64
- let int = mantissa >> (64 - shl);
- let fraction = ((mantissa as u64) << shl) as f64 * FRACT_SCALE;
- (int, fraction)
- } else if shl < 128 {
- // x << 64..128
- let int = mantissa << (shl - 64);
- (int, 0.0)
- } else {
- // x << 128..
- (0, 0.0)
- }
-}
-
-fn join_u16_into_f64(bits: [u16; 4]) -> f64 {
- let u64_bits = (bits[0] as u64)
- | ((bits[1] as u64) << 16)
- | ((bits[2] as u64) << 32)
- | ((bits[3] as u64) << 48);
-
- f64::from_bits(u64_bits)
-}
-
-fn get_max(a: i32, b: i32) -> i32 {
- a.max(b)
-}
-
-/// Converts a float via multiplication and truncation
-fn to_multiply_and_truncate(number: f64, mul: i32) -> i64 {
- (number * mul as f64) as i64
-}
-
-fn to_median_filter(data: &Vec) -> Vec {
- let mut filtered = Vec::with_capacity(data.len());
- // 10minutes of data
- let mut filter = Filter::new(50);
- for point in data {
- let point_int = MetricTag::QuasiRandom.from_float(*point);
- let median = filter.consume(point_int);
- filtered.push(median)
- }
- filtered
-}
-
-/// Check the type of metric and tag it
-fn tag_metric(filename: &str) -> MetricTag {
- // Should sort this by the probability of each tag, so the ones that are more common are dealt first
- // If it says percent_ or _utilization
- let mut regex = Regex::new(r"(?m)percent_|_utilization").unwrap();
- if regex.captures(filename).is_some() {
- // 2 significant digits resolution (Linux resolution)
- return MetricTag::Percent(100);
- }
- // if it says _client_request
- regex = Regex::new(r"(?m)_client_request").unwrap();
- if regex.captures(filename).is_some() {
- // Fractional requests are nothing but an averaging artifact
- return MetricTag::NotFloat;
- }
- // if it says _seconds
- regex = Regex::new(r"(?m)_seconds").unwrap();
- if regex.captures(filename).is_some() {
- // 1 micro second resolution
- return MetricTag::Duration(1_000_000);
- }
- // if it says _seconds
- regex = Regex::new(r"(?m)_networkindelta|_networkoutdelta|_heapmemoryused_").unwrap();
- if regex.captures(filename).is_some() {
- return MetricTag::QuasiRandom;
- }
- MetricTag::Other
-}
-
-/// Go through the data, check min and max values, DC Component
-/// Check if data fits in 8,16,24,32 bits. If so reduce it to a single channel with
-/// those bit depths.
-fn analyze_data(data: &Vec) -> (i32, i64, bool) {
- let mut min: f64 = 0.0;
- let mut max: f64 = 0.0;
- let mut fractional = false;
- for value in data {
- let t_value = *value;
- if split_n(t_value).1 != 0.0 {
- fractional = true;
- }
- if t_value > max {
- max = t_value
- };
- if t_value < min {
- min = t_value
- };
- }
- // Check max size of values
- // For very large numbers (i32 and i64), it might be ideal to detect the dc component
- // of the signal. And then remove it later
- let max_int = split_n(max).0; // This is the DC component
- let min_int = split_n(min).0;
-
- // If fractional is it relevant?
- let max_frac = split_n(max).1;
-
- // Finding the bitdepth without the DC component
- let recommended_bitdepth = find_bitdepth(max_int - min_int, min_int);
- if !fractional {
- info!(" Recommended Bitdepth: {} ", recommended_bitdepth);
- } else {
- info!(
- " Fractional, Recommended Bitdepth: {}, Fractions max: {}",
- recommended_bitdepth, max_frac
- );
- }
- (recommended_bitdepth, min_int, fractional)
-}
-
-fn analyze_int_data(data: &Vec) -> (i32, i64) {
- let mut min: i64 = 0;
- let mut max: i64 = 0;
- for value in data {
- let t_value = *value;
- if t_value > max {
- max = t_value
- };
- if t_value < min {
- min = t_value
- };
- }
-
- let recommended_bitdepth = find_bitdepth(max - min, min);
- info!(" Recommended Bitdepth: {} ", recommended_bitdepth);
- (recommended_bitdepth, min)
-}
-
-fn find_bitdepth(max_int: i64, min_int: i64) -> i32 {
- // Check where those ints fall into
- let bitdepth = match max_int {
- _ if max_int <= u8::MAX.into() => 8,
- _ if max_int <= i16::MAX.into() => 16,
- _ if max_int <= i32::MAX.into() => 32,
- _ => 64,
- };
-
- let bitdepth_signed = match min_int {
- _ if min_int == 0 => 8,
- _ if min_int >= i16::MIN.into() => 16,
- _ if min_int >= i32::MIN.into() => 32,
- _ => 64,
- };
-
- get_max(bitdepth, bitdepth_signed)
-}
-
-fn process_args(input_path: &str, arguments: &Args) {
- if arguments.directory {
- handle_directory(input_path, arguments);
- } else {
- process_file(input_path.into(), arguments, None);
- }
-}
-
-fn handle_directory(input_path: &str, arguments: &Args) {
- let new_directory = format!("new_{}", input_path);
-
- if fs::create_dir_all(&new_directory).is_err() {
- error!("Unable to create directory: {}", new_directory);
- return;
- }
-
- if let Ok(entries) = fs::read_dir(input_path) {
- for entry_result in entries {
- match entry_result {
- Ok(entry) if entry.path().is_file() => {
- process_file(entry.path(), arguments, Some(&new_directory));
- }
- Err(e) => error!("Error reading directory entry: {}", e),
- _ => {}
- }
- }
- } else {
- error!("Error reading directory: {}", input_path);
- }
-}
-fn process_file(full_path: PathBuf, arguments: &Args, new_directory: Option<&str>) {
- if let Some(filename) = full_path.file_name().and_then(|s| s.to_str()) {
- let output_path = construct_output_path(filename, new_directory);
- let mut file = match File::create(&output_path) {
- Ok(file) => file,
- Err(_) => {
- error!("Unable to create file: {}", output_path);
- return;
- }
- };
-
- process_data_and_write_output(&full_path, &mut file, arguments);
- }
-}
-
-fn construct_output_path(filename: &str, new_directory: Option<&str>) -> String {
- match new_directory {
- Some(dir) => format!("{}/new_{}.txt", dir, filename),
- None => format!("new_{}.txt", filename),
- }
-}
-
-fn process_data_and_write_output(full_path: &Path, file: &mut File, arguments: &Args) {
- let full_path_str = full_path.to_str().unwrap_or("");
- debug!("File: {} ,", full_path_str);
- let mut _bitdepth = 64;
- let mut _dc_component: i64 = 0;
- let mut _fractional = true;
- let wav_data = read_metrics_from_wav(full_path_str);
- if arguments.dump_raw {
- writeln!(file, "{:?}", wav_data).expect("Unable to write to file");
- }
- // Depending on Metric Tag, apply a transformation
- let tag = tag_metric(full_path_str);
- debug!("Tag: {:?}", tag);
- let iwav_data = match tag {
- MetricTag::Other => Vec::new(),
- MetricTag::QuasiRandom => to_median_filter(&wav_data),
- _ => wav_data.iter().map(|x| tag.from_float(*x)).collect(),
- };
- // We split the code here
- if !iwav_data.is_empty() {
- _fractional = false;
- if arguments.dump_optimized {
- writeln!(file, "{:?}", iwav_data).expect("Unable to write to file");
- }
- (_bitdepth, _dc_component) = analyze_int_data(&iwav_data);
- } else {
- (_bitdepth, _dc_component, _fractional) = analyze_data(&wav_data);
- }
- if _bitdepth == 64 || _fractional {
- debug!("No optimization, exiting");
- std::process::exit(0);
- } else if arguments.write {
- debug!("Writing optimal file!");
- match iwav_data.len() {
- 0 => write_optimal_wav(full_path_str, wav_data, _bitdepth, _dc_component, 1),
- _ => write_optimal_int_wav(full_path_str, iwav_data, _bitdepth, _dc_component, 1),
- }
- }
-}
-#[derive(Parser, Default, Debug)]
-#[command(author, version, about, long_about = None)]
-struct Args {
- /// input file
- input: String,
-
- /// Write a new file with optimized settings, named filename_OPT.wav
- #[arg(short)]
- write: bool,
-
- #[arg(short, action)]
- directory: bool,
-
- /// Samplerate to generate the optimized file
- #[arg(short, long)]
- samplerate: Option,
-
- /// Write raw (original) samples to a file, named as raw.out
- #[arg(long, action)]
- dump_raw: bool,
-
- /// Write optimized samples to a file, named as optimized.out
- #[arg(long, action)]
- dump_optimized: bool,
-}
-
-fn main() {
- // How to break the float part??? --> THERE ARE NO FLOATS!
- // https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/s2-proc-stat
- env_logger::init();
- let arguments = Args::parse();
- debug!("{:?}", arguments);
- process_args(&arguments.input, &arguments);
-}
diff --git a/plot_comparison.sh b/plot_comparison.sh
index 8c868a3..9082bd2 100755
--- a/plot_comparison.sh
+++ b/plot_comparison.sh
@@ -4,22 +4,22 @@ error=$1
cp ../../wbro-july/$filename.wbro tmp.wbro
-target/debug/brro-compressor --compressor fft --error $error --verbose tmp.wbro > ../../comparison-$filename.m
-target/debug/brro-compressor -u --verbose tmp.bro >> ../../comparison-$filename.m
+target/debug/atsc --compressor fft --error $error --verbose tmp.wbro > ../../comparison-$filename.m
+target/debug/atsc -u --verbose tmp.bro >> ../../comparison-$filename.m
sed -i -e 's/Output/output_fft/g' ../../comparison-$filename.m
cp ../../wbro-july/$filename.wbro tmp.wbro
-target/debug/brro-compressor --compressor idw --error $error --verbose tmp.wbro > /dev/null
-target/debug/brro-compressor -u --verbose tmp.bro >> ../../comparison-$filename.m
+target/debug/atsc --compressor idw --error $error --verbose tmp.wbro > /dev/null
+target/debug/atsc -u --verbose tmp.bro >> ../../comparison-$filename.m
sed -i -e 's/Output/output_idw/g' ../../comparison-$filename.m
cp ../../wbro-july/$filename.wbro tmp.wbro
-target/debug/brro-compressor --compressor polynomial --error $error --verbose tmp.wbro > /dev/null
-target/debug/brro-compressor -u --verbose tmp.bro >> ../../comparison-$filename.m
+target/debug/atsc --compressor polynomial --error $error --verbose tmp.wbro > /dev/null
+target/debug/atsc -u --verbose tmp.bro >> ../../comparison-$filename.m
sed -i -e 's/Output/output_poly/g' ../../comparison-$filename.m
diff --git a/plot_data.sh b/plot_data.sh
index 6d21482..219c4c1 100755
--- a/plot_data.sh
+++ b/plot_data.sh
@@ -2,8 +2,8 @@
filename=$2
compressor=$1
cp ../../wbro-july/$filename.wbro tmp.wbro
-target/debug/brro-compressor --compressor $compressor --error 1 --verbose tmp.wbro > ../../$filename-$compressor.m
-target/debug/brro-compressor -u --verbose tmp.bro >> ../../$filename-$compressor.m
+target/debug/atsc --compressor $compressor --error 1 --verbose tmp.wbro > ../../$filename-$compressor.m
+target/debug/atsc -u --verbose tmp.bro >> ../../$filename-$compressor.m
echo "plot(Input,'b', Output,'r')" >> ../../$filename-$compressor.m
echo "print -dpng $filename.png" >> ../../$filename-$compressor.m
rm tmp.wbro
diff --git a/prometheus-remote/Cargo.toml b/prometheus-remote/Cargo.toml
deleted file mode 100644
index d2a13a8..0000000
--- a/prometheus-remote/Cargo.toml
+++ /dev/null
@@ -1,25 +0,0 @@
-[package]
-name = "prometheus-remote"
-version = "0.1.0"
-authors = ["Carlos Rolo "]
-edition = "2021"
-license = "Apache-2.0"
-description = "Remote Read/Write server for prometheus"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-[dependencies]
-hound = "3.5"
-chrono = "0.4.26"
-claxon = "0.4.3"
-warp = "0.3.5"
-tokio = { version= "1", features = ["full"] }
-symphonia = { version = "0.5.3", features = ["flac"] }
-prom-remote-api = { version = "0.3.0", features = ["warp"] }
-async-trait = "0.1.71"
-env_logger = "0.11.0"
-log = "0.4.0"
-clap = {version = "4.3.14", features = ["derive"] }
-regex = "1.9.1"
-median = "0.3.2"
-dtw_rs = "0.9.5"
-
diff --git a/prometheus-remote/src/flac_reader.rs b/prometheus-remote/src/flac_reader.rs
deleted file mode 100644
index a87c1b5..0000000
--- a/prometheus-remote/src/flac_reader.rs
+++ /dev/null
@@ -1,288 +0,0 @@
-use std::fs::File;
-
-use symphonia::core::audio::SampleBuffer;
-use symphonia::core::codecs::{Decoder, DecoderOptions};
-use symphonia::core::errors::Error as SymphoniaError;
-use symphonia::core::formats::{FormatOptions, FormatReader};
-use symphonia::core::io::MediaSourceStream;
-use symphonia::core::meta::MetadataOptions;
-use symphonia::core::probe::Hint;
-
-use chrono::{DateTime, Utc};
-
-use crate::lib_vsri;
-
-// --- Flac Reader
-// Remote Reader Spec: ?
-
-/* --- File Structure STRUCTURE
-note: t=point in time, chan = channel, samples are the bytes for each channel.
- in this example, each sample is made of 2 bytes (16bit)
-+---------------------------+---------------------------+-----
-| Frame 1 | Frame 2 | etc
-+-------------+-------------+-------------+-------------+-----
-| chan 1 @ t1 | chan 2 @ t1 | chan 1 @ t2 | chan 2 @ t2 | etc
-+------+------+------+------+------+------+------+------+-----
-| byte | byte | byte | byte | byte | byte | byte | byte | etc
-+------+------+------+------+------+------+------+------+-----
- */
-// TODO: Read from WAV file
-// Flac metric is giving a ton of issues, trying to get something simpler
-pub struct SimpleFlacReader {
- file: File, // The File where the metric is
-}
-
-impl SimpleFlacReader {
- pub fn new(file: File, _start_ts: i64) -> Self {
- SimpleFlacReader { file }
- }
-
- pub fn get_samples(
- &self,
- start: Option,
- end: Option,
- ) -> std::result::Result, SymphoniaError> {
- let mut sample_vec: Vec = Vec::new();
- let mut reader = claxon::FlacReader::new(&self.file).unwrap();
- let channels = reader.streaminfo().channels;
- let mut sample_count = 0;
- // TODO: Make this hold up to channel number
- let mut sample_channel_data: [u16; 4] = [0, 0, 0, 0];
- let mut frame_reader = reader.blocks();
- let mut block = claxon::Block::empty();
- loop {
- // Read a single frame. Recycle the buffer from the previous frame to
- // avoid allocations as much as possible.
- match frame_reader.read_next_or_eof(block.into_buffer()) {
- Ok(Some(next_block)) => block = next_block,
- Ok(None) => break, // EOF.
- Err(error) => panic!("[DEBUG][READ][FLAC] {}", error),
- }
- debug!(
- "[READ][SimpleFLaC] Processing block... Samples processed: {:?}",
- sample_count
- );
- if sample_count < start.unwrap_or(0) {
- continue;
- }
- if sample_count > end.unwrap_or(lib_vsri::MAX_INDEX_SAMPLES) {
- continue;
- }
- for sample in 0..block.duration() {
- for channel in 0..channels {
- sample_channel_data[channel as usize] = block.sample(channel, sample) as u16;
- }
- sample_vec.push(SimpleFlacReader::join_u16_into_f64(sample_channel_data));
- sample_count += 1;
- }
- }
- debug!(
- "[READ][SimpleFLaC] Returning samples for interval: {} {} Sample count: {:?}",
- start.unwrap_or(0),
- end.unwrap_or(0),
- sample_count
- );
- Ok(sample_vec)
- }
-
- pub fn get_all_samples(&self) -> std::result::Result, SymphoniaError> {
- self.get_samples(None, None)
- }
-
- fn join_u16_into_f64(bits: [u16; 4]) -> f64 {
- let u64_bits = (bits[0] as u64)
- | ((bits[1] as u64) << 16)
- | ((bits[2] as u64) << 32)
- | ((bits[3] as u64) << 48);
-
- f64::from_bits(u64_bits)
- }
-}
-
-pub struct FlacMetric {
- timeseries_data: Vec<(i64, f64)>, // Sample Data
- file: File, // The File where the metric is
- interval_start: i64, // The start interval in timestamp with miliseconds
- decoder: Option>, // Flac decoder
- format_reader: Option>, // Flac format reader
-}
-
-impl FlacMetric {
- pub fn new(file: File, start_ts: i64) -> Self {
- FlacMetric {
- timeseries_data: Vec::new(),
- file,
- interval_start: start_ts,
- decoder: None,
- format_reader: None,
- }
- }
-
- fn datetime_from_ms(real_time: i64) -> String {
- // Time is in ms, convert it to seconds
- let datetime = DateTime::::from_timestamp(real_time / 1000, 0).unwrap();
- // Transform datetime to string with the format YYYY-MM-DD
- let datetime_str = datetime.format("%Y-%m-%d").to_string();
- datetime_str
- }
-
- /// Load sample data into the Flac Object
- fn load_samples(self) -> Vec<(i64, f64)> {
- Vec::new()
- }
-
- fn get_format_reader(&self) -> Box {
- // TODO: One more unwrap to deal with
- let owned_file = self.file.try_clone().unwrap();
- debug!("[READ][FLAC] Probing file: {:?}", owned_file);
- let file = Box::new(owned_file);
- // Create the media source stream using the boxed media source from above.
- let mss = MediaSourceStream::new(file, Default::default());
- // Use the default options when reading and decoding.
- let format_opts: FormatOptions = Default::default();
- let metadata_opts: MetadataOptions = Default::default();
- // Probe the media source stream for a format.
- let probed = symphonia::default::get_probe()
- .format(
- Hint::new().mime_type("FLaC"),
- mss,
- &format_opts,
- &metadata_opts,
- )
- .unwrap();
- // Get the format reader yielded by the probe operation.
- probed.format
- }
-
- fn get_decoder(&self) -> Box {
- let decoder_opts: DecoderOptions = Default::default();
- let format = self.get_format_reader();
- // Get the default track.
- let track = format.default_track().unwrap();
- // Create a decoder for the track.
- symphonia::default::get_codecs()
- .make(&track.codec_params, &decoder_opts)
- .unwrap()
- }
-
- /// Read samples from a file with an optional start and end point.
- pub fn get_samples(
- &self,
- start: Option,
- end: Option,
- ) -> std::result::Result, SymphoniaError> {
- let mut sample_vec: Vec = Vec::new();
- let mut format_reader = self.get_format_reader();
- let mut decoder = self.get_decoder();
- let channels = decoder.codec_params().channels.unwrap().count();
- let mut sample_buf = None;
- let mut frame_counter: i32 = 0;
- let start_frame = start.unwrap_or(0);
- let end_frame = end.unwrap_or(lib_vsri::MAX_INDEX_SAMPLES);
- // Loop over all the packets, get all the samples and return them
- loop {
- let packet = match format_reader.next_packet() {
- Ok(packet) => packet,
- Err(err) => break error!("[READ]Reader error: {}", err),
- };
- // How many frames inside the packet
- let dur = packet.dur() as i32;
- // Check if we need to decode this packet or not
- if !(start_frame < frame_counter + dur && end_frame > frame_counter + dur) {
- continue;
- }
- // Decode the packet into samples.
- // TODO: This is overly complex, split into its own code
- match decoder.decode(&packet) {
- Ok(decoded) => {
- // Consume the decoded samples (see below).
- if sample_buf.is_none() {
- // Get the audio buffer specification.
- let spec = *decoded.spec();
- // Get the capacity of the decoded buffer. Note: This is capacity, not length!
- let duration = decoded.capacity() as u64;
- // Create the sample buffer.
- sample_buf = Some(SampleBuffer::::new(duration, spec));
- }
- // Each frame contains several samples, we need to get the frame not the sample. Since samples = frames * channels
- if let Some(buf) = &mut sample_buf {
- buf.copy_interleaved_ref(decoded);
- let mut i16_samples: [u16; 4] = [0, 0, 0, 0];
- let mut i = 1; // Starting at 1, channel number is not 0 indexed...
- for sample in buf.samples() {
- if i >= channels {
- frame_counter += 1;
- if frame_counter >= start_frame && frame_counter <= end_frame {
- sample_vec.push(FlacMetric::join_u16_into_f64(i16_samples));
- }
- i = 1;
- }
- i16_samples[i - 1] = *sample as u16;
- i += 1;
- }
- }
- }
- Err(SymphoniaError::DecodeError(err)) => error!("[READ]Decode error: {}", err),
- Err(err) => break error!("[READ]Unexpeted Decode error: {}", err),
- }
- }
- Ok(sample_vec)
- }
-
- /// Read all samples from a file
- pub fn get_all_samples(&self) -> std::result::Result, SymphoniaError> {
- let mut sample_vec: Vec = Vec::new();
- let mut format_reader = self.get_format_reader();
- let mut decoder = self.get_decoder();
- let channels = decoder.codec_params().channels.unwrap().count();
- let mut sample_buf = None;
- // Loop over all the packets, get all the samples and return them
- loop {
- let packet = match format_reader.next_packet() {
- Ok(packet) => packet,
- Err(err) => break debug!("[READ]Reader error: {}", err),
- };
- // Decode the packet into audio samples.
- match decoder.decode(&packet) {
- Ok(decoded) => {
- // Consume the decoded audio samples (see below).
- if sample_buf.is_none() {
- // Get the audio buffer specification.
- let spec = *decoded.spec();
- // Get the capacity of the decoded buffer. Note: This is capacity, not length!
- let duration = decoded.capacity() as u64;
- // Create the sample buffer.
- sample_buf = Some(SampleBuffer::::new(duration, spec));
- }
- if let Some(buf) = &mut sample_buf {
- buf.copy_interleaved_ref(decoded);
- let mut i16_samples: [u16; 4] = [0, 0, 0, 0];
- let mut i = 1; // Starting at 1, channel number is not 0 indexed...
- for sample in buf.samples() {
- if i >= channels {
- sample_vec.push(FlacMetric::join_u16_into_f64(i16_samples));
- i = 1;
- }
- i16_samples[i - 1] = *sample as u16;
- i += 1;
- }
- }
- }
- Err(SymphoniaError::DecodeError(err)) => error!("[READ]Decode error: {}", err),
- Err(err) => break error!("[READ]Unexpeted Decode error: {}", err),
- }
- }
- // Just to make it compile
- Ok(sample_vec)
- }
-
- /// Recreate a f64
- fn join_u16_into_f64(bits: [u16; 4]) -> f64 {
- let u64_bits = (bits[0] as u64)
- | ((bits[1] as u64) << 16)
- | ((bits[2] as u64) << 32)
- | ((bits[3] as u64) << 48);
-
- f64::from_bits(u64_bits)
- }
-}
diff --git a/prometheus-remote/src/fs_utils.rs b/prometheus-remote/src/fs_utils.rs
deleted file mode 100644
index 9ca62f2..0000000
--- a/prometheus-remote/src/fs_utils.rs
+++ /dev/null
@@ -1,414 +0,0 @@
-use chrono::{DateTime, Duration, Utc};
-/// All the utils/code related the to file management
-///
-/// ASSUMPTION: EACH DAY HAS 1 FILE!!! If this assumption change, change this file!
-/// TODO: (BIG ONE!) Make this time period agnostic (so it would work with days, weeks, etc)
-/// For a READ request that needs data for MetricX from Ta to Tb this would do the following:
-/// 1. Do we have metricX? -> No, stop.
-/// 2. Which file has Ta, and which has Tb?
-/// 2.1 Select them to read
-/// 3. Read the indexes, and retrieve the available samples
-///
-/// Suggested internal Data Structure of the WAV file
-///
-/// +--------------------------------------------------------------------------------------------------------+
-/// | HEADER | i16 @Chan1 | i16 @Chan2 | i16 @Chan3 | i16 @Chan4 | tick @Chan5 | i16 @Chan1 | i16 @Chan2 |...|
-/// +--------------------------------------------------------------------------------------------------------+
-///
-/// Prometheus Point: f64 split into 4x i16 (channel 1 to 4) Timestamp: Tick into Channel 5
-///
-use std::fs::{self, File};
-use std::mem;
-
-use crate::flac_reader::SimpleFlacReader;
-use crate::lib_vsri::{day_elapsed_seconds, start_day_ts, Vsri, MAX_INDEX_SAMPLES};
-
-struct DateRange(DateTime, DateTime);
-
-// Iterator for Day to Day
-// TODO: move this to several impl? So we can return iterators over several time periods?
-impl Iterator for DateRange {
- type Item = DateTime;
- fn next(&mut self) -> Option {
- if self.0 <= self.1 {
- let next = self.0 + Duration::days(1);
- Some(mem::replace(&mut self.0, next))
- } else {
- None
- }
- }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct PromDataPoint {
- pub point: f64,
- pub time: i64,
-}
-
-impl PromDataPoint {
- /// Creates a new Prometheus Data Point. It assumes a timestamp with seconds since EPOCH, and converts internally to
- /// miliseconds since EPOCH.
- pub fn new(data: f64, timestamp: i64) -> Self {
- PromDataPoint {
- point: data,
- time: timestamp * 1000,
- }
- }
-}
-/// Holds a time range for the file and index
-#[derive(Debug, Clone, Copy)]
-struct FileTimeRange {
- start: i32,
- end: i32,
-}
-
-impl FileTimeRange {
- fn new(start: i32, end: i32) -> Self {
- FileTimeRange { start, end }
- }
-}
-
-/// A struct that allows the precise location of data inside the file is in it
-/// TODO: Make FILE a FlacReader
-#[derive(Debug)]
-pub struct DataLocator {
- file: File,
- index: Vsri,
- time_range: FileTimeRange,
- date: DateTime,
-}
-
-impl DataLocator {
- /// Creates a new DataLocator, includes the File, Index and the Time Range for the data it is expected to return.
- /// This is a lazy, doesn't check for the intersection between the time range and data owned until the data is
- /// requested.
- fn new(file: File, index: Vsri, time_range: FileTimeRange, date: DateTime) -> Self {
- DataLocator {
- file,
- index,
- time_range,
- date,
- }
- }
-
- /// Checks if the Locator time_range intersects with the Index data
- fn do_intersect(&self) -> bool {
- // If the data start after the end of the range or the data ends before the beggining of the range
- if self.index.min() > self.time_range.end || self.index.max() < self.time_range.start {
- return false;
- }
- // index function checks for no ownership, this function checks for ownership, invert the result
- !self
- .index
- .is_empty([self.time_range.start, self.time_range.end])
- }
-
- fn get_samples_from_range(&self) -> Option<[i32; 2]> {
- // By default, get all the samples
- let mut sample_range: [i32; 2] = [0, MAX_INDEX_SAMPLES];
- if !self.do_intersect() {
- return None;
- }
- match self.time_range.start {
- 0 => {
- sample_range[0] = 0;
- }
- _ => {
- // There is intersection, it can unwrap safely
- sample_range[0] = self.index.get_this_or_next(self.time_range.start).unwrap();
- }
- }
- match self.time_range.end {
- // Match cannot shadow statics and whatever
- _ if self.time_range.end == MAX_INDEX_SAMPLES => {
- sample_range[1] = self.index.get_sample_count();
- }
- _ => {
- // There is intersection, it can unwrap safely
- sample_range[1] = self
- .index
- .get_this_or_previous(self.time_range.start)
- .unwrap();
- }
- }
- Some(sample_range)
- }
-
- /// Consumes the DataLocator to return a Vec of PromDataPoints
- pub fn into_prom_data_point(self) -> Vec {
- let mut prom_data = Vec::new();
- let samples_locations = self.get_samples_from_range();
- let flac_metric = SimpleFlacReader::new(self.file, self.time_range.start as i64);
- let tmp_vec = self.index.get_all_timestamps();
- // There goes an empty arry
- if samples_locations.is_none() {
- return prom_data;
- }
- let start = samples_locations.unwrap()[0];
- let end = samples_locations.unwrap()[1] - 1;
- debug!(
- "[READ] Samples located! From {} to {}. TS available: {}",
- start,
- end,
- tmp_vec.len()
- );
- let time_for_samples = &tmp_vec[start as usize..=end as usize];
- // The time I learned if..else is an expression!
- let temp_result = if start == 0 && end == self.index.get_sample_count() {
- flac_metric.get_all_samples()
- } else {
- flac_metric.get_samples(Some(start), Some(end))
- };
- match temp_result {
- // Pack this into DataPoints
- Ok(samples) => {
- for (v, t) in samples.into_iter().zip(time_for_samples.iter()) {
- let ts = *t as i64 + start_day_ts(self.date);
- prom_data.push(PromDataPoint::new(v, ts * 1000));
- }
- }
- Err(err) => {
- error!("[READ] Error processing FLaC file {:?}", err);
- return prom_data;
- }
- }
- prom_data
- }
-
- /// Given a metric name and a time interval, returns all the files handles for the files that *might* contain that data (No data range intersection is done here)
- pub fn get_locators_for_range(
- metric_name: &str,
- start_time: i64,
- end_time: i64,
- ) -> Option> {
- let mut file_index_vec = Vec::new();
- let data_locator_vec: Vec;
- let start_date = DateTime::::from_timestamp(start_time / 1000, 0).unwrap();
- let end_date = DateTime::::from_timestamp(end_time / 1000, 0).unwrap();
- let file_time_intervals = time_intervals(start_time, end_time);
- debug!(
- "[READ] Time intervals for the range {:?} ",
- file_time_intervals
- );
- let mut range_count = 0;
- for date in DateRange(start_date, end_date).enumerate() {
- let data_file_name = format!("{}_{}", metric_name, date.1.format("%Y-%m-%d"));
- debug!(
- "[READ] Time intervals for file {}: {:?} ",
- data_file_name, file_time_intervals[range_count]
- );
- let vsri = Vsri::load(&data_file_name);
- range_count += 1;
- let file = match fs::File::open(format!("{}.flac", data_file_name.clone())) {
- Ok(file) => file,
- Err(err) => {
- warn!(
- "[READ] Error processing {}.flac. Error: {}. Skipping file.",
- data_file_name, err
- );
- continue;
- }
- };
- // If I got here, I should be able to unwrap Vsri safely.
- file_index_vec.push((file, vsri.unwrap(), date));
- }
- // Creating the Time Range array
- let start_ts_i32 = day_elapsed_seconds(start_time);
- let end_ts_i32 = day_elapsed_seconds(end_time);
- let mut time_intervals = Vec::new();
- match range_count {
- 1 => {
- time_intervals.push(FileTimeRange::new(start_ts_i32, end_ts_i32));
- }
- 2 => {
- time_intervals.push(FileTimeRange::new(start_ts_i32, MAX_INDEX_SAMPLES));
- time_intervals.push(FileTimeRange::new(0, end_ts_i32));
- }
- _ => {
- time_intervals.push(FileTimeRange::new(start_ts_i32, MAX_INDEX_SAMPLES));
- for _i in 2..range_count {
- time_intervals.push(FileTimeRange::new(0, MAX_INDEX_SAMPLES));
- }
- time_intervals.push(FileTimeRange::new(0, end_ts_i32));
- }
- }
-
- // We have at least one file create the Object
- if !file_index_vec.is_empty() {
- data_locator_vec = file_index_vec
- .into_iter()
- .map(|item| DataLocator::new(item.0, item.1, time_intervals[item.2 .0], item.2 .1))
- .collect();
- debug!("[READ] Returning Object {:?} ", data_locator_vec);
- return Some(data_locator_vec);
- }
- None
- }
-}
-
-/// Returns a Vector of array of time intervals (in seconds) for the interval of time
-fn time_intervals(start_time: i64, end_time: i64) -> Vec<[i32; 2]> {
- let mut time_intervals = Vec::new();
- let start_date = DateTime::::from_timestamp(start_time / 1000, 0).unwrap();
- let end_date = DateTime::::from_timestamp(end_time / 1000, 0).unwrap();
- let start_ts_i32 = day_elapsed_seconds(start_time);
- let end_ts_i32 = day_elapsed_seconds(end_time);
- let date_spread_size = DateRange(start_date, end_date).count();
- match date_spread_size {
- 1 => {
- time_intervals.push([start_ts_i32, end_ts_i32]);
- }
- 2 => {
- time_intervals.push([start_ts_i32, MAX_INDEX_SAMPLES]);
- time_intervals.push([0, end_ts_i32]);
- }
- _ => {
- time_intervals.push([start_ts_i32, MAX_INDEX_SAMPLES]);
- for _i in 2..date_spread_size {
- time_intervals.push([0, MAX_INDEX_SAMPLES]);
- }
- time_intervals.push([0, end_ts_i32]);
- }
- }
- time_intervals
-}
-
-/// Given a metric name and a time interval, returns all the files handles for the files that contain that data
-pub fn get_file_index_time(
- metric_name: &str,
- start_time: i64,
- end_time: i64,
-) -> Option> {
- DataLocator::get_locators_for_range(metric_name, start_time, end_time)
-}
-
-pub fn data_locator_into_prom_data_point(data: Vec) -> Vec {
- debug!("[READ] Locators: {:?}", data);
- let mut data_points = Vec::new();
- for dl in data {
- let mut proms = dl.into_prom_data_point();
- if !proms.is_empty() {
- data_points.append(&mut proms);
- }
- }
- data_points
-}
-
-/// Retrieves all the available data points in a timerange in the provided Vector of files and indexes
-pub fn get_data_between_timestamps(
- start_time: i64,
- end_time: i64,
- file_vec: Vec<(File, Vsri)>,
-) -> Vec {
- let mut data_points = Vec::new();
- /* Processing logic:
- Case 1 (2+ files):
- The first file, the period if from `start_time` to end of the file (use index),
- The second until the last file (exclusive), we need all the data points we can get (read full file).
- The last file we need from start until the `end_time` (use index).
- Case 2 (Single file):
- Read the index to locate the start sample and the end sample.
- Read the file and obtain said samples.
- */
- // How many files to process
- let file_count = file_vec.len();
- // Get the baseline timestamps to add to the index timestamps
- let start_date = DateTime::::from_timestamp(start_time / 1000, 0).unwrap();
- let end_date = DateTime::::from_timestamp(end_time / 1000, 0).unwrap();
- let ts_bases: Vec = DateRange(start_date, end_date).map(start_day_ts).collect();
- let start_ts_i32 = day_elapsed_seconds(start_time);
- let end_ts_i32 = day_elapsed_seconds(end_time);
- // Files might not match the intervals of time, a time array of time intervals need to be done.
-
- // Where the samples land in the indexes
- let mut samples_locations: [i32; 2];
- for pack in file_vec.into_iter().enumerate() {
- let iter_index = pack.0;
- let file = pack.1 .0;
- let vsri = pack.1 .1;
- debug!(
- "[READ] Locating samples. VSRI {:?} TS: {} - {}",
- vsri, start_ts_i32, end_ts_i32
- );
- // Check if the timestamps intercept the index space
- if file_count == 1 {
- debug!("[READ] Processing single file...");
- // Case 2
- // get_sample can return None
- if vsri.min() > end_ts_i32 || vsri.max() < start_ts_i32 {
- debug!("[READ] No intersection. Returning.");
- return data_points;
- }
- let start_sample = vsri.get_this_or_next(start_ts_i32);
- if start_sample.is_none() {
- // No sample in the file fits the current requested interval
- debug!("[READ] No intersection (Part2). Returning.");
- return data_points;
- }
- // If I can start reading the file, I can get at least one sample, so it is safe to unwrap.
- let end_sample = vsri.get_this_or_previous(end_ts_i32).unwrap();
- samples_locations = [start_sample.unwrap(), end_sample];
- } else {
- // Case 1
- debug!("[READ] Processing multiple files...");
- match pack.0 {
- // First file
- 0 => {
- let start_sample = vsri.get_this_or_next(start_ts_i32);
- if start_sample.is_none() {
- continue;
- }
- samples_locations = [start_sample.unwrap(), vsri.get_sample_count()];
- }
- // Last file
- _ if iter_index == file_count - 1 => {
- let end_sample = vsri.get_this_or_previous(end_ts_i32);
- if end_sample.is_none() {
- continue;
- }
- samples_locations = [0, end_sample.unwrap()];
- }
- // Other files
- _ => {
- // Collect the full file
- samples_locations = [0, vsri.get_sample_count()];
- }
- }
- }
- // Collect the data points
- let flac_metric = SimpleFlacReader::new(file, start_time);
- let tmp_vec = vsri.get_all_timestamps();
- let start = samples_locations[0];
- let end = samples_locations[1] - 1;
- debug!(
- "[READ] Samples located! From {} to {}. TS available: {}",
- start,
- end,
- tmp_vec.len()
- );
- // !@)(#*&!@)# usize and ints...
- let time_for_samples = &tmp_vec[start as usize..=end as usize];
- // The time I learned if..else is an expression!
- let temp_result = if start == 0 && end == vsri.get_sample_count() {
- flac_metric.get_all_samples()
- } else {
- flac_metric.get_samples(Some(start), Some(end))
- };
-
- match temp_result {
- // Pack this into DataPoints
- Ok(samples) => {
- for (v, t) in samples.into_iter().zip(time_for_samples.iter()) {
- let ts = *t as i64 + ts_bases[iter_index];
- data_points.push(PromDataPoint::new(v, ts));
- }
- }
- Err(err) => {
- error!("[READ] Error processing FLaC file {:?}", err);
- continue;
- }
- }
- }
- debug!("[READ] Returning datapoints: {:?}", data_points);
- data_points
-}
diff --git a/prometheus-remote/src/lib_vsri.rs b/prometheus-remote/src/lib_vsri.rs
deleted file mode 100644
index 8acec72..0000000
--- a/prometheus-remote/src/lib_vsri.rs
+++ /dev/null
@@ -1,470 +0,0 @@
-use chrono::{DateTime, Timelike, Utc};
-/// Very Small Rolo Index
-/// This is an index made for detection of gaps in continuous data with the same sampling rate.
-/// Each continuous segment of data will be mapped to a line using the formula y = mx + B plus
-/// the number of points in the data series.
-/// m - Sampling rate
-/// b - Series initial point in time in [x,y]
-/// x - sample # in the data file, this is ALWAYS sequential. There are no holes in samples
-/// y - time
-///
-/// This way, discovering the segment number is solving the above equation for X if the
-/// time provided is bigger than the initial point.
-///
-/// best case for sample retrieval O(1)
-/// worst case O(N) (N is the number of segments)
-/// Space usage: 5Bytes for 64k samples.
-/// Or: 30Bytes for 2^32 Samples
-///
-/// Example of content of an index
-/// 55745
-/// 59435
-/// 15,0,55745,166
-/// 15,166,58505,63
-use std::fs::File;
-use std::io::{BufRead, BufReader, BufWriter, Write};
-
-// TODO: This should be configurable. Indexes are build for 1 day worth of samples, at 1 sample per second
-pub static MAX_INDEX_SAMPLES: i32 = 86400;
-
-// Helper functions, this should be moved somewhere
-/// Returns the number of seconds elapsed for the day provided in the `timestamp_sec`
-pub fn day_elapsed_seconds(timestamp_sec: i64) -> i32 {
- let datetime = DateTime::::from_timestamp(timestamp_sec, 0).unwrap();
- // Extract the time components (hour, minute, and second) from the DateTime
- let hour = datetime.time().hour();
- let minute = datetime.time().minute();
- let second = datetime.time().second();
- // Calculate the total seconds since the start of the day
- (hour * 3600 + minute * 60 + second) as i32
-}
-
-/// Returns the timestamp for the beginning of the day given a DateTime object.
-pub fn start_day_ts(dt: DateTime) -> i64 {
- let hour = dt.time().hour();
- let minute = dt.time().minute();
- let second = dt.time().second();
- dt.timestamp() - (hour * 3600 + minute * 60 + second) as i64
-}
-
-/// In this implementation we are writing sample by sample to the WAV file, so
-/// we can't do a proper segment calculation. So there will a special first segment
-/// that will hold the first point so we can calculate the segments from there.
-///
-/// # Examples
-/// Creating a new index, metric is of expected time 0, but for sure location of X is 0
-/// ```no_run
-/// let vsri = Vsri::new("metric_name", 0, 0);
-/// vsri.flush();
-/// ```
-/// Updating an index, adding point at time 5sec
-/// ```no_run
-/// let vsri = Vsri::load("metric_name").unwrap().update_for_point(5);
-/// vsri.flush();
-/// ```
-/// Fetch a sample location from the index given a timestamp
-/// ```no_run
-/// let vsri = Vsri::load("metric_name").unwrap();
-/// vsri.get_sample_location("metric_name", 5);
-/// ```
-
-/// Index Structure
-/// index_name: Name of the index file we are indexing
-/// min_ts: the minimum TS available in this file
-/// max_ts: the highest TS available in this file
-/// vsri_segments: Description of each segment
-/// [sample_rate (m), initial_point(x,y), # of samples(length)]
-/// Each segments describes a line with the form of mX + B that has a lenght
-/// of # of samples.
-#[derive(Debug)]
-pub struct Vsri {
- index_file: String,
- min_ts: i32,
- max_ts: i32,
- // TODO: ENUM here to make it simpler to understand what each point in the array means
- vsri_segments: Vec<[i32; 4]>, // [Sample Rate (m), X0, Y0, # of Samples]
-}
-
-impl Vsri {
- /// Creates the index, it doesn't create the file in the disk
- /// flush needs to be called for that
- pub fn new(filename: &String) -> Self {
- debug!("[INDEX] Creating new index!");
- let segments: Vec<[i32; 4]> = Vec::new();
- Vsri {
- index_file: filename.to_string(),
- min_ts: 0,
- max_ts: 0,
- vsri_segments: segments,
- }
- }
-
- /// Given a filename and a time location, returns the sample location in the
- /// data file. Or None in case it doesn't exist.
- pub fn get_sample_location(filename: String, y: i32) -> Option {
- let vsri = match Vsri::load(&filename) {
- Ok(vsri) => vsri,
- Err(_err) => return None,
- };
- if vsri.min() <= y && y <= vsri.max() {
- return vsri.get_sample(y);
- }
- None
- }
-
- /// Get the sample for this timestamp or the next one
- pub fn get_this_or_next(&self, y: i32) -> Option {
- let r = self.get_sample(y).or(self.get_next_sample(y));
- debug!("[INDEX] This or next location {:?} for TS {}", r, y);
- r
- }
-
- /// Get the sample for this timestamp or the previous one
- pub fn get_this_or_previous(&self, y: i32) -> Option {
- let r = self.get_sample(y).or(self.get_previous_sample(y));
- debug!("[INDEX] This or previous location {:?} for TS {}", r, y);
- r
- }
-
- /// Returns the next sample for the provided timestamp.
- /// This might be useful to find the next segment timestamp if the timestamp
- /// is in between segments. It will return None in case the timestamp is over
- /// the maximum timestamp of the index.
- pub fn get_next_sample(&self, y: i32) -> Option {
- if y < self.min() {
- return Some(0);
- } else if y >= self.max() {
- return None;
- }
- // It wasn't smaller, so let's see if we have a sample that matches
- for segment in self.vsri_segments.clone().into_iter().rev() {
- let first_sample = segment[1];
- let y0 = segment[2];
- if y <= y0 {
- return Some(first_sample);
- }
- }
- None
- }
-
- /// Returns the previous sample for the provided timestamp.
- /// This might be useful to find the previous segment timestamp if the timestamp
- /// is in between segments. It will return None in case the timestamp is bellow
- /// the minimum timestamp of the index.
- pub fn get_previous_sample(&self, y: i32) -> Option {
- if y < self.min() {
- return None;
- } else if y >= self.max() {
- // Return the last segment, # of samples. That is the total # of samples in a file
- return Some(self.get_sample_count());
- }
- // Cycle through the segments
- for segment in &self.vsri_segments {
- let first_sample = segment[1];
- let y0 = segment[2];
- if y < y0 {
- // Return the last sample of the previous segment
- return Some(first_sample - 1);
- }
- }
- None
- }
-
- /// Checks if the time segment provided falls in an empty space (Between 2 segments)
- /// This is useful to check intersections. If this function returns false the provided
- /// time segment does overlap with the existing time segments in the file
- pub fn is_empty(&self, time_segment: [i32; 2]) -> bool {
- // I could simple try to get 2 samples and if one of the returns, it is not empty
- // but I would walk segments twice instead of once
- match &self.vsri_segments.len() {
- 1 => {
- // It starts or ends inside the segment (might be a single sample)
- if (time_segment[0] >= self.min() && time_segment[0] <= self.max())
- || (time_segment[1] <= self.max() && time_segment[1] >= self.min())
- {
- return false;
- }
- // Or it contains the whole segment
- if time_segment[0] < self.min() && time_segment[1] > self.max() {
- return false;
- }
- }
- _ => {
- // More than 1 segment
- let mut previous_seg_end: i32 = 0;
- for (segment_count, segment) in self.vsri_segments.iter().enumerate() {
- let sample_rate = segment[0];
- let y0 = segment[2];
- let num_samples = segment[3];
- let segment_end_y = y0 + (sample_rate * (num_samples - 1));
- // If we are in the 2+ segment, lets test if the time falls in the middle
- if segment_count >= 1
- && (time_segment[0] > previous_seg_end && time_segment[1] < y0)
- {
- return true;
- }
- // Could this be simplified with Karnaugh map? I'll dig my books later
- // It starts or ends inside the segment
- if (time_segment[0] >= y0 && time_segment[0] < segment_end_y)
- || (time_segment[1] < segment_end_y && time_segment[1] >= y0)
- {
- return false;
- }
- // Or it contains the whole segment
- if time_segment[0] < y0 && time_segment[1] > segment_end_y {
- return false;
- }
- // At this point, time segments doesn't touch this segment.
- previous_seg_end = segment_end_y;
- }
- }
- }
- // Didn't find any intersection, or left in the middle, it is empty
- true
- }
-
- /// Update the index for the provided point
- /// y - time in seconds
- pub fn update_for_point(&mut self, y: i32) -> Result<(), ()> {
- // Y needs to be bigger that the current max_ts, otherwise we are appending a point in the past
- // TODO: #11 Quantiles sends several metrics for the same time, how to handle it?
- if y < self.max_ts {
- // Is this always a period (day) change? Assuming so
- warn!(
- "[INDEX] Trying to index a point in the past: {}, provided point: {}",
- self.max_ts, y
- );
- return Err(());
- }
- self.max_ts = y;
- let segment_count = self.vsri_segments.len();
- // Empty segments, create a new one, this is also a new index, update the timestamps
- if segment_count == 0 {
- self.min_ts = y;
- self.vsri_segments.push(self.create_fake_segment(y));
- return Ok(());
- }
- if self.is_fake_segment() {
- // In the presence of a fake segment (where m is 0), and a new point, we are now
- // in a situation we can calculate a decent segment
- self.vsri_segments[segment_count - 1] = self.generate_segment(y);
- } else {
- // Check ownership by the current segment
- if self.fits_segment(y) {
- // It fits, increase the sample count and it's done
- debug!("[INDEX] Same segment, updating. TS: {}", y);
- self.vsri_segments[segment_count - 1][3] += 1;
- return Ok(());
- }
- // If it doesn't fit, create a new fake segment
- self.vsri_segments.push(self.create_fake_segment(y));
- }
- Ok(())
- }
-
- /// Minimum time stamp
- pub fn min(&self) -> i32 {
- self.min_ts
- }
-
- /// Maximum time stamp
- pub fn max(&self) -> i32 {
- self.max_ts
- }
-
- fn calculate_b(&self, segment: &[i32; 4]) -> i32 {
- // b = y - mx
-
- segment[2] - segment[0] * segment[1]
- }
-
- /// Returns the most recent (the last) calculated segment
- fn current_segment(&self) -> [i32; 4] {
- match self.vsri_segments.len() {
- 0 => [0, 0, 0, 0],
- _ => self.vsri_segments[self.vsri_segments.len() - 1],
- }
- }
-
- /// Get the sample location for a given point in time, or None if there is no sample for that specific TS
- pub fn get_sample(&self, y: i32) -> Option {
- for segment in &self.vsri_segments {
- let sample_rate = segment[0];
- let y0 = segment[2];
- let num_samples = segment[3];
-
- let segment_end_y = y0 + (sample_rate * (num_samples - 1));
-
- if y >= y0 && y <= segment_end_y {
- // x = (y - b)/ m
- // TODO: This can return floats!
- let x_value = (y - self.calculate_b(segment)) / sample_rate;
- return Some(x_value);
- }
- }
- None // No matching segment found for the given Y value
- }
-
- /// For a given sample position, return the timestamp associated
- pub fn get_time(&self, x: i32) -> Option {
- match x {
- 0 => Some(self.min()),
- _ if x > self.get_sample_count() => None,
- _ if x == self.get_sample_count() => Some(self.max()),
- // it is somewhere in the middle
- _ => {
- // Find the segment where X fits
- for segment in &self.vsri_segments {
- if x >= segment[1] && x < (segment[1] + segment[3]) {
- // Belongs here! Return Segment TS + the TS interval * x
- let y = segment[2] + segment[0] * x;
- return Some(y);
- }
- continue;
- }
- None
- }
- }
- }
-
- /// Returns a vector will all the timestamps covered by this index
- pub fn get_all_timestamps(&self) -> Vec {
- let mut time_vec = Vec::new();
- for segment in &self.vsri_segments {
- let samples = segment[3]; // Range is EXCLUSIVE above
- let time_step = segment[0];
- let initial_ts = segment[2];
- let time_iter = (0..samples).map(|f| (f * time_step) + initial_ts);
- time_vec.extend(time_iter);
- }
- time_vec
- }
-
- pub fn get_sample_count(&self) -> i32 {
- let last_segment = self.current_segment();
- last_segment[3] + last_segment[1]
- }
-
- /// Generates a segment from a point. It uses information stored in the segment
- /// to regenerate the same segment with the new point information.
- fn generate_segment(&self, y: i32) -> [i32; 4] {
- // Retrieve the last segment
- let last_segment = self.current_segment();
- // double check for correctness
- if last_segment[0] != 0 {
- return last_segment;
- }
- // Calculate the new segment
- // m = (y1-y0)/(x1-x0) -> (x1-x0) = 1 => m = y1-y0 (X is a sequence)
- let m = y - last_segment[2];
- // We got m, the initial points are the same, and now we have 2 samples
- [m, last_segment[1], last_segment[2], 2]
- }
-
- fn update_segment_samples(mut self) {
- let segment_count = self.vsri_segments.len();
- self.vsri_segments[segment_count - 1][3] += 1;
- }
-
- /// Generate a fake segment, this can't be used for ownership testing
- /// x is the previous segment sample number
- /// We only have the first y0 point, nothing else
- fn create_fake_segment(&self, y: i32) -> [i32; 4] {
- debug!("[INDEX] New segment, creating for point: {}", y);
- let segment = self.current_segment();
- // First point of the new segment: Prior starting point + Number of samples
- let x = segment[1] + segment[3];
- [0, x, y, 1]
- }
-
- /// Checks if the most recent segment is a fake segment
- fn is_fake_segment(&self) -> bool {
- let last_segment = self.current_segment();
- last_segment[0] == 0
- }
-
- /// Returns true if a point fits the last segment of the index
- fn fits_segment(&self, y: i32) -> bool {
- let last_segment = self.current_segment();
- let b = self.calculate_b(&last_segment);
- // What we have to check, is with the given y, calculate x.
- // Then check if x fits the interval for the current line
- // and it has to be the next one in the line
- // x = (y - b)/ m
- // TODO: Can return float, watch out
- let x_value = (y - b) / last_segment[0];
- debug!(
- "[INDEX] Fit Calculation (Segment {:?}). b: {}, x: {}, calculated x: {}",
- last_segment,
- b,
- (last_segment[3] + last_segment[1]),
- x_value
- );
- x_value == last_segment[3] + last_segment[1]
- }
-
- /// Writes the index to the disk
- /// File format
- /// line | content
- /// 1 | minimum timestamp on this file. eg: 10
- /// 2 | maximum timestamp on this file. eg: 34510
- /// 3..N | Segments. 4 fields separated by commas. ex: 0,1,2,3
- pub fn flush(&self) -> Result<(), std::io::Error> {
- let file = File::create(format!("{}.vsri", &self.index_file))?;
- let mut writer = BufWriter::new(file);
-
- // Write index_file, min_ts, max_ts on the first three lines
- writeln!(writer, "{}", self.min_ts)?;
- writeln!(writer, "{}", self.max_ts)?;
-
- // Write each vsri_segment on a separate line
- for segment in &self.vsri_segments {
- writeln!(
- writer,
- "{},{},{},{}",
- segment[0], segment[1], segment[2], segment[3]
- )?;
- }
-
- writer.flush()?;
- Ok(())
- }
-
- /// Reads an index file and loads the content into the structure
- /// TODO: Add error control (Unwrap hell)
- pub fn load(filename: &String) -> Result {
- debug!("[INDEX] Load existing index");
- let file = File::open(format!("{}.vsri", &filename))?;
- let reader = BufReader::new(file);
- let mut min_ts = 0;
- let mut max_ts = 0;
- let mut segments: Vec<[i32; 4]> = Vec::new();
- let mut i = 1; // Line 1,2 are not segments.
- for line in reader.lines() {
- let line = line?;
- match i {
- 1 => {
- min_ts = line.trim().parse::().unwrap();
- }
- 2 => {
- max_ts = line.trim().parse::().unwrap();
- }
- _ => {
- let values = line
- .split(',')
- .map(|value| value.trim().parse::())
- .collect::, _>>()
- .unwrap();
- segments.push([values[0], values[1], values[2], values[3]]);
- }
- }
- i += 1;
- }
- Ok(Vsri {
- index_file: filename.to_string(),
- min_ts,
- max_ts,
- vsri_segments: segments,
- })
- }
-}
diff --git a/prometheus-remote/src/main.rs b/prometheus-remote/src/main.rs
deleted file mode 100644
index dd36cb2..0000000
--- a/prometheus-remote/src/main.rs
+++ /dev/null
@@ -1,247 +0,0 @@
-// Lucas - Once the project is far enough along I strongly reccomend reenabling dead code checks
-#![allow(dead_code)]
-
-mod flac_reader;
-mod fs_utils;
-mod lib_vsri;
-mod wav_writer;
-use fs_utils::data_locator_into_prom_data_point;
-use wav_writer::WavMetric;
-
-use async_trait::async_trait;
-use std::{convert::Infallible, sync::Arc};
-
-use prom_remote_api::{
- types::{
- Error, Label, MetricMetadata, Query, QueryResult, RemoteStorage, Result, Sample,
- TimeSeries, WriteRequest,
- },
- web,
-};
-use warp::Filter;
-
-use log::{debug, error, info, warn};
-
-#[macro_use]
-extern crate log;
-
-use crate::fs_utils::get_file_index_time;
-
-// Data sampling frequency. How many seconds between each sample.
-static VERSION: &str = "0.1.1";
-
-fn get_flac_samples_to_prom(
- metric: &str,
- source: &str,
- _job: &str,
- start_ms: i64,
- end_ms: i64,
- step_ms: i64,
-) -> Vec {
- // TODO: #6 Count the number of samples for the given metric! -> Can be done with the Index alone \m/ \m/
- // TODO: #1 Do not ignore Job!
- // TODO: #2 Do not ignore Step!
- // Just for today, Step in the files is always 15sec, 15000 ms.
- let sample_step = (step_ms / 15000) as usize;
- if step_ms == 0 {
- return vec![Sample {
- value: 1.0,
- timestamp: start_ms,
- }];
- }
- // Build the metric name
- let metric_name = format!("{}_{}", metric, source);
- let files_to_parse = get_file_index_time(&metric_name, start_ms, end_ms);
- if files_to_parse.is_none() {
- error!("No data found!");
- return vec![Sample {
- value: 1.0,
- timestamp: start_ms,
- }];
- }
- //let prom_vec = get_data_between_timestamps(start_ms, end_ms, files_to_parse.unwrap());
- let prom_vec = data_locator_into_prom_data_point(files_to_parse.unwrap());
- let prom_len = prom_vec.len();
- //debug!("[MAIN] Prom data points: {:?}", prom_vec);
- debug!(
- "[MAIN] Returned samples: {:?} Requested Step: {:?} Proposed Steps: {:?}",
- prom_len, step_ms, sample_step
- );
- // Convert into Samples and apply step_ms
- //let mut out = Vec::new();
- //let mut prev_sample_ts: i64 = 0;
- /*
- for (i, pdp) in prom_vec.into_iter().enumerate() {
- if i == 0 {
- out.push(Sample{value: pdp.point, timestamp: pdp.time});
- prev_sample_ts = pdp.time;
- continue;
- }
- if pdp.time < prev_sample_ts + step_ms { continue; }
- out.push(Sample{value: pdp.point, timestamp: pdp.time});
- prev_sample_ts = pdp.time;
- }
- debug!("[MAIN] Requested Step: {:?} Proposed Steps: {:?} Original len {:?} Final len {:?}", step_ms, sample_step, prom_len, out.len());
-
- out */
- prom_vec
- .iter()
- .map(|pdp| Sample {
- value: pdp.point,
- timestamp: pdp.time,
- })
- .collect()
- //prom_vec.iter().step_by(sample_step).map(|pdp| Sample{value: pdp.point, timestamp: pdp.time}).collect()
- //let flac_content = get_flac_samples(metric, start_ms, end_ms).unwrap();
- // Flac reader is ignoring step returning way to many samples. So we have to deal with step here
- // Transforming the result into Samples
- //let step_size: usize = (step_ms/DATA_INTERVAL_MSEC).try_into().unwrap();
- //debug!(" # of FLaC samples: {} Step size ms: {} Internal step: {}", flac_content.len(), step_ms, step_size);
- //flac_content.iter().step_by(step_size).enumerate().map(|(i, sample)| Sample{value: *sample as f64, timestamp: (start_ms + (i as i64)*step_ms) as i64}).collect()
-}
-
-fn parse_remote_write_request(
- timeseries: &TimeSeries,
- metadata: Option<&MetricMetadata>,
-) -> Result<()> {
- debug!("[WRITE] samples: {:?}", timeseries.samples);
- debug!("[WRITE] labels: {:?}", timeseries.labels);
- debug!("[WRITE] metadata: {:?}", metadata);
-
- let mut metric: Option<&str> = None;
- let mut source: Option<&str> = None;
- let mut job: Option<&str> = None;
-
- for label in ×eries.labels {
- match label.name.as_str() {
- "__name__" => metric = Some(&label.value),
- "instance" => source = Some(&label.value),
- "job" => job = Some(&label.value),
- _ => (),
- }
- }
-
- if let (Some(metric), Some(source), Some(job)) = (metric, source, job) {
- // Not going to share state, flush it once you're done.
- // TODO: #3 Improve write performance (?)
- let mut metric_data: Vec<(i64, f64)> = timeseries
- .samples
- .iter()
- .map(|x| (x.timestamp, x.value))
- .collect();
- if timeseries.samples.is_empty() {
- error!("[WRITE][MAIN] Empty samples: {:?}", timeseries.samples);
- return Ok(());
- }
- let mut wav_metric = WavMetric::new(
- metric.to_string(),
- source.to_string(),
- job.to_string(),
- metric_data[0].0,
- );
- let mutable_metric_data = &mut metric_data;
- wav_metric.add_bulk_timeseries(mutable_metric_data);
- match wav_metric.flush() {
- Ok(_) => return Ok(()),
- Err(_samples) => {
- // TODO: Improve this situation... (Retry?)
- return Ok(());
- }
- }
- } else {
- warn!("[WRITE] Missing metric or source");
- }
- Ok(())
-}
-
-#[derive(Clone, Copy)]
-struct FlacStorage;
-
-impl FlacStorage {
- fn with_context() -> impl Filter + Clone {
- warp::any().map(|| 1)
- }
-}
-
-#[async_trait]
-impl RemoteStorage for FlacStorage {
- type Err = Error;
- type Context = u64;
- // TODO: Figure out why the empty Results
- async fn write(&self, _ctx: Self::Context, req: WriteRequest) -> Result<()> {
- trace!("[MAIN][WRITE] req:{req:?}");
- if req.metadata.is_empty() {
- for timeseries in req.timeseries {
- let _ = parse_remote_write_request(×eries, None);
- //break;
- }
- } else {
- for (timeseries, metadata) in req.timeseries.iter().zip(req.metadata.iter()) {
- let _ = parse_remote_write_request(timeseries, Some(metadata));
- }
- }
- Ok(())
- }
-
- async fn process_query(&self, _ctx: &Self::Context, query: Query) -> Result {
- debug!("[MAIN] flac read, req:{query:?}");
- let metric = &query.matchers[0].value;
- // TODO: Get these values from somewhere else
- let job = "flac-remote";
- let instance = "localhost:9090";
- Ok(QueryResult {
- timeseries: vec![TimeSeries {
- labels: vec![
- Label {
- name: "job".to_string(),
- value: job.to_string(),
- },
- Label {
- name: "instance".to_string(),
- value: instance.to_string(),
- },
- Label {
- name: "__name__".to_string(),
- value: metric.to_string(),
- },
- ],
- samples: get_flac_samples_to_prom(
- metric,
- instance,
- job,
- query.start_timestamp_ms,
- query.end_timestamp_ms,
- query
- .hints
- .as_ref()
- .map(|hint| hint.step_ms)
- .unwrap_or(1000),
- ),
- ..Default::default()
- }],
- })
- }
-}
-
-#[tokio::main(flavor = "current_thread")]
-// BIG TODO: Make the code configurable (loads of hardcoded stuff)
-async fn main() {
- env_logger::init();
- info!("FFT-Storage v. {}", VERSION);
- let storage = Arc::new(FlacStorage);
- let write_api = warp::path!("write")
- .and(web::warp::with_remote_storage(storage.clone()))
- .and(FlacStorage::with_context())
- .and(web::warp::protobuf_body())
- .and_then(web::warp::write);
- let query_api = warp::path!("read")
- .and(web::warp::with_remote_storage(storage))
- .and(FlacStorage::with_context())
- .and(web::warp::protobuf_body())
- .and_then(web::warp::read);
-
- let routes = warp::path("api").and(write_api.or(query_api));
- let port = 9201;
- info!("Server up, listening on {} port {}", "127.0.0.1", port);
- warp::serve(routes).run(([127, 0, 0, 1], port)).await;
-}
diff --git a/prometheus-remote/src/wav_writer.rs b/prometheus-remote/src/wav_writer.rs
deleted file mode 100644
index ee8c997..0000000
--- a/prometheus-remote/src/wav_writer.rs
+++ /dev/null
@@ -1,237 +0,0 @@
-use chrono::{DateTime, Utc};
-use hound::{WavSpec, WavWriter};
-use std::fs::File;
-use std::fs::{metadata, OpenOptions};
-use std::process::Command;
-
-use crate::lib_vsri::{day_elapsed_seconds, Vsri};
-
-// --- Write layer
-// Remote write spec: https://prometheus.io/docs/concepts/remote_write_spec/
-pub struct WavMetric {
- pub metric_name: String, // Metric name provided by prometheus
- pub instance: String, // Instance name provided by prometheus
- pub job: String, // Job name provided by prometheus
- pub timeseries_data: Vec<(i64, f64)>, // Sample Data
- pub creation_time: String, // The timestamp that this structure was created.
- pub last_file_created: Option, // Name of the last file created, !! might not make sense anymore !!
-}
-// Here is where things get tricky. Either we have a single strutcure and implement several WavWriters or we segment at the metric collection level.
-// The advantage of implementing at the writing level is that we can look into the data and make a better guess based on the data.
-// There is also the problem of not understanding the data clearly, or not having the WHOLE data available and making assumptions on
-// a incomplete dataset.
-// Another way we can/should get around this would be "hinting" for the data type.
-// If we are dealing with percentages we can go with i16, etc.
-// Option B, less optimal, but more functional, is breaking f64 in 16bit parts and storing each part in its own channel.
-// We are choosing option B!
-
-impl WavMetric {
- /// Create a new WavMetric struct. `start_sample_ts` *must be* timestamp with miliseconds!
- pub fn new(name: String, source: String, job: String, start_sample_ts: i64) -> WavMetric {
- // Sample needs to fall within the file that the TS refers too, not the calendar day
- let start_date = DateTime::::from_timestamp(start_sample_ts / 1000, 0).unwrap();
- // TODO: Do not ignore JOB!
- WavMetric {
- metric_name: name,
- instance: source,
- job,
- timeseries_data: Vec::new(),
- creation_time: start_date.format("%Y-%m-%d").to_string(),
- last_file_created: None,
- }
- }
- /// Flushes the metric to a WAV file
- // TODO: Unwrap hell in here. Need better error control
- // Too many assumptions on correct behavior of all the code. Assumption is the mother of all... Needs to be fixed
- pub fn flush(mut self) -> Result<(), i32> {
- let mut processed_samples: i32 = 0;
- let vsri: Option;
- if self.timeseries_data.is_empty() {
- // Can't flush empty data
- error!("[WRITE][WAV] Call flush on empty data");
- return Err(processed_samples);
- }
- // Append if file exists, otherwise create spec and flush a new file
- let mut wav_writer = match self.last_file_created.is_none() {
- true => {
- let handlers = self.create_file().unwrap();
- vsri = Some(handlers.1);
- handlers.0
- }
- false => {
- let file = OpenOptions::new()
- .write(true)
- .read(true)
- .open(self.last_file_created.unwrap())
- .unwrap();
- // Load the index file
- // TODO: one more unwrap to work on later
- vsri = Some(Vsri::load(&self.metric_name).unwrap());
- WavWriter::new_append(file).unwrap()
- }
- };
- // TODO: #12 Check if the timestamp is one day ahead, if so, create another file, and pack the previous one as FLAC
- let vsri_unwrapped = &mut vsri.unwrap();
- let mut int_r: Result<(), i32> = Ok(());
- for (ts, sample) in self.timeseries_data {
- let short_ts = ts / 1000;
- let r = vsri_unwrapped.update_for_point(day_elapsed_seconds(short_ts));
- if r.is_err() {
- // Period changed (default: day)
- warn!("[WRITE][WAV] Detected a day change while processing samples. Wrote {} before error.", processed_samples);
- int_r = Err(processed_samples);
- break;
- }
- let channel_data = WavMetric::split_f64_into_i16s(sample);
- // Write the samples interleaved
- for sample in channel_data {
- let ww = wav_writer.write_sample(sample);
- if ww.is_err() {
- error!(
- "[WAVWRITER] Unable to write sample {:?} in file {:?}!",
- sample, self.metric_name
- );
- return Err(processed_samples);
- }
- }
- processed_samples += 1;
- }
- debug!("[WRITE][WAV] Wrote {} samples", processed_samples);
- // TODO: Process there errors too, create different errors here
- let r = vsri_unwrapped.flush();
- if r.is_err() {
- error!(
- "[WAVWRITER] Unable to flush VSRI for {:?}!",
- self.metric_name
- );
- panic!(
- "[WAVWRITER] Failed flushing index. Lost information. {}",
- r.unwrap_err()
- )
- }
- let r2 = wav_writer.finalize();
- if r2.is_err() {
- error!(
- "[WAVWRITER] Unable to flush WAV file {:?}!",
- self.metric_name
- );
- panic!(
- "[WAVWRITER] Failed flushing file. Lost information. {}",
- r.unwrap_err()
- )
- }
- int_r
- }
-
- /// Create a file accordingly to the day of the year, the metric and the instance that generated the metric
- /// TODO: Create file shouldn't open a file for append. Should only create. Fix this (or rename)
- fn create_file(&mut self) -> Result<(WavWriter, Vsri), hound::Error> {
- let spec = WavMetric::generate_wav_header(None);
- let file_name = format!(
- "{}_{}_{}",
- self.metric_name, self.instance, self.creation_time
- );
- let file_path = format!("./{}.wav", file_name);
- // Create a new WAV file, if exists or open the existing one
- if let Ok(meta) = metadata(&file_path) {
- if meta.is_file() {
- let file = OpenOptions::new().write(true).read(true).open(&file_path)?;
- let wav_writer = WavWriter::new_append(file)?;
- return Ok((wav_writer, Vsri::load(&file_name).unwrap()));
- }
- }
- let file = File::create(&file_path)?;
- let wav_writer = WavWriter::new(file, spec)?;
- self.last_file_created = Some(file_path);
- // TODO: Y can't be 0. Needs to be TS
- Ok((wav_writer, Vsri::new(&file_name)))
- }
-
- /// Generate the WAV file header.
- fn generate_wav_header(channels: Option) -> WavSpec {
- hound::WavSpec {
- channels: channels.unwrap_or(4) as u16,
- sample_rate: 8000,
- bits_per_sample: 16,
- sample_format: hound::SampleFormat::Int,
- }
- }
-
- /// Add a single metric value to the structure
- pub fn add_timeseries(mut self, ts: i64, value: f64) {
- self.timeseries_data.push((ts, value))
- }
-
- /// Add a vector of data to the existing timeseries
- pub fn add_bulk_timeseries(&mut self, timeseries: &mut Vec<(i64, f64)>) {
- self.timeseries_data.append(timeseries)
- }
-
- /// Read a range in the structure
- pub fn get_range(self, ts_start: i64, ts_end: i64) -> Vec<(i64, f64)> {
- let mut i = 0;
- let mut j = 0;
- for (count, (ts, _)) in self.timeseries_data.iter().enumerate() {
- if *ts < ts_start {
- i = count
- }
- if *ts < ts_end {
- j = count;
- break;
- }
- }
- if i > 0 {
- return self.timeseries_data[i - 1..j].to_vec();
- }
- self.timeseries_data[..j].to_vec()
- }
-
- /// Instead of chasing data types and converting stuff, let's just unpack the f64 and
- /// put it into different channels. This way we can always garantee a clean i16 Wave file
- fn split_f64_into_i16s(value: f64) -> [i16; 4] {
- let bits: u64 = value.to_bits();
-
- let i16_1 = (bits & 0xFFFF) as i16;
- let i16_2 = ((bits >> 16) & 0xFFFF) as i16;
- let i16_3 = ((bits >> 32) & 0xFFFF) as i16;
- let i16_4 = ((bits >> 48) & 0xFFFF) as i16;
-
- [i16_1, i16_2, i16_3, i16_4]
- }
-
- /// Recreate a f64
- fn create_f64_from_16bits(bits: [u16; 4]) -> f64 {
- let u64_bits = (bits[0] as u64)
- | ((bits[1] as u64) << 16)
- | ((bits[2] as u64) << 32)
- | ((bits[3] as u64) << 48);
-
- f64::from_bits(u64_bits)
- }
-
- /// Rotate the wav file after the interval and save it as a FLaC file
- fn rotate_wav_into_flac(self) {
- let file_in = format!(
- "{}_{}_{}.wav",
- self.metric_name, self.instance, self.creation_time
- );
- let file_out = format!(
- "{}_{}_{}.flac",
- self.metric_name, self.instance, self.creation_time
- );
- // Command: sox input.wav output.flac
- let output = Command::new("sox")
- .arg(file_in)
- .arg(file_out)
- .output()
- .expect("Error converting WAV to FLAC");
- if !output.status.success() {
- panic!("Could not rotate file!")
- }
- }
-
- /// Check if the current timestamp is within the file period
- fn is_ts_valid(_ts: i64) -> bool {
- true
- }
-}
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 55a49f1..20e20c7 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,3 +1,3 @@
[toolchain]
-channel = "1.80"
+channel = "1.81"
components = [ "rustfmt", "clippy" ]
diff --git a/tools/src/bin/dwt_finder.rs b/tools/src/bin/dwt_finder.rs
deleted file mode 100644
index 14fc480..0000000
--- a/tools/src/bin/dwt_finder.rs
+++ /dev/null
@@ -1,66 +0,0 @@
-use clap::Parser;
-use dtw_rs::{Algorithm, DynamicTimeWarping, ParameterizedAlgorithm};
-
-fn read_metrics_from_wav(filename: &str) -> Vec {
- let r_reader = hound::WavReader::open(filename);
- let mut reader = match r_reader {
- Ok(reader) => reader,
- Err(_err) => {
- return Vec::new();
- }
- };
- let num_channels = reader.spec().channels as usize;
-
- let mut raw_data: Vec = Vec::new();
- let mut u64_holder: [u16; 4] = [0, 0, 0, 0];
-
- // Iterate over the samples and channels and push each sample to the vector
- let mut current_channel: usize = 0;
- for sample in reader.samples::() {
- u64_holder[current_channel] = sample.unwrap() as u16;
- current_channel += 1;
- if current_channel == num_channels {
- raw_data.push(join_u16_into_f64(u64_holder));
- current_channel = 0;
- }
- }
- raw_data
-}
-
-fn join_u16_into_f64(bits: [u16; 4]) -> f64 {
- let u64_bits = (bits[0] as u64)
- | ((bits[1] as u64) << 16)
- | ((bits[2] as u64) << 32)
- | ((bits[3] as u64) << 48);
-
- f64::from_bits(u64_bits)
-}
-
-#[derive(Parser, Default, Debug)]
-struct Arguments {
- /// First wav file
- file_one: String,
- /// Second wav file
- file_two: String,
- /// Distance
- distance: usize,
- /// Block size
- block: usize,
-}
-
-fn main() {
- let args = Arguments::parse();
- println!("{:?}", args);
-
- let binding_a: Vec = read_metrics_from_wav(&args.file_one);
- let binding_b: Vec = read_metrics_from_wav(&args.file_two);
- let vec_slices_a: Vec<&[f64]> = binding_a.chunks(args.block).collect();
- let vec_slices_b: Vec<&[f64]> = binding_b.chunks(args.block).collect();
- let data_a = vec_slices_a[0];
- let data_b = vec_slices_b[0];
-
- let param = dtw_rs::Restriction::Band(args.distance);
- let dtw = DynamicTimeWarping::with_param(data_a, data_b, param);
-
- println!("Path: {:?}, Distance: {}", dtw.path(), dtw.distance());
-}
diff --git a/tools/src/bin/flac_reader_tester.rs b/tools/src/bin/flac_reader_tester.rs
deleted file mode 100644
index 007b383..0000000
--- a/tools/src/bin/flac_reader_tester.rs
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
-This file compares a FLAC and WAV file and if the content is identical
-Also good to test if the FLAC and WAV read routines are good
-*/
-
-/* Read a WAV file, */
-fn _read_metrics_from_wav(filename: &str) -> Vec {
- let mut reader = hound::WavReader::open(filename).unwrap();
- let num_samples = reader.len() as usize / reader.spec().channels as usize;
- let num_channels = reader.spec().channels as usize;
-
- // Create a vector to hold the audio data
- let mut samples = Vec::with_capacity(num_samples * num_channels);
-
- // Iterate over the samples and channels and push each sample to the vector
- for sample in reader.samples::() {
- samples.push(sample.unwrap());
- }
- samples
-}
-
-/* Read a FLAC file */
-fn read_metrics_from_flac(filename: &str) -> Vec {
- let mut reader = claxon::FlacReader::open(filename).unwrap();
- // Create a vector to hold the audio data
- let mut samples = Vec::with_capacity(reader.streaminfo().samples.unwrap() as usize);
- for sample in reader.samples() {
- samples.push(sample.unwrap() as u16);
- }
- samples
-}
-
-fn read_metrics_from_flac_by_bloc(filename: &str) -> Vec {
- let mut sample_vec: Vec = Vec::new();
- let mut reader = claxon::FlacReader::open(filename).unwrap();
- let channels = reader.streaminfo().channels as usize;
- let mut sample_channel_data = vec![0u16; channels];
- let mut frame_reader = reader.blocks();
- let mut block = claxon::Block::empty();
-
- loop {
- // Read a single frame. Recycle the buffer from the previous frame to
- // avoid allocations as much as possible.
- match frame_reader.read_next_or_eof(block.into_buffer()) {
- Ok(Some(next_block)) => block = next_block,
- Ok(None) => break, // EOF.
- Err(error) => panic!("[DEBUG][READ][FLAC] {}", error),
- }
- for sample in 0..block.duration() {
- #[allow(clippy::needless_range_loop)]
- for channel in 0..channels {
- sample_channel_data[channel] = block.sample(channel as u32, sample) as u16;
- }
-
- // Process the sample_channel_data as needed
- for &sample in &sample_channel_data {
- sample_vec.push(sample);
- }
-
- // Optionally, can print debug information
- println!(
- "Sample {}/{}, Channels: {:?}",
- sample,
- block.duration(),
- &sample_channel_data
- );
- }
- }
- sample_vec
-}
-
-fn _read_metrics_from_flac_in_interval(filename: &str, start: u32, end: u32) -> Vec {
- let mut reader = claxon::FlacReader::open(filename).unwrap();
- // Create a vector to hold the audio data
- let start_sample = start * reader.streaminfo().sample_rate;
- let end_sample = end * reader.streaminfo().sample_rate;
- //let mut samples = Vec::with_capacity(reader.streaminfo().samples.unwrap() as usize);
- let mut samples: Vec = Vec::new();
- for (i, sample) in reader.samples().enumerate() {
- let i = i as u32;
- if start_sample <= i && i <= end_sample {
- samples.push(sample.unwrap() as i16);
- } else if i > end_sample {
- break;
- }
- }
- samples
-}
-
-fn main() {
- println!("Testing, does FLAC reading is the same as WAV?");
- let _filename = "2023-05-11_15-11-19.wav";
- let filename_flac =
- "/home/crolo/code/prom_data/go_memstats_frees_total_localhost:9090_2023-07-07.flac";
- let _filename_flac_single = "3_single_channel.flac";
- //let samples = read_metrics_from_wav(filename);
- //println!("{:?}", samples);
- let samples_flac = read_metrics_from_flac(filename_flac);
- let samples_flac_b = read_metrics_from_flac_by_bloc(filename_flac);
- println!("{:?}", samples_flac);
- println!("{:?}", samples_flac_b);
- assert_eq!(samples_flac_b, samples_flac);
- //let samples_flac_in_interval = read_metrics_from_flac_in_interval(filename_flac, 5, 7);
- println!("Sample Flac {:?}", samples_flac.len());
- println!("Sample Flac {:?}", samples_flac_b.len());
-}
diff --git a/tools/src/bin/mid_channel_computing.rs b/tools/src/bin/mid_channel_computing.rs
deleted file mode 100644
index 61d674b..0000000
--- a/tools/src/bin/mid_channel_computing.rs
+++ /dev/null
@@ -1,112 +0,0 @@
-use std::fs::File;
-
-use clap::Parser;
-use hound::{WavSpec, WavWriter};
-
-fn read_metrics_from_wav(filename: &str) -> Vec {
- let r_reader = hound::WavReader::open(filename);
- let mut reader = match r_reader {
- Ok(reader) => reader,
- Err(_err) => {
- return Vec::new();
- }
- };
- let num_channels = reader.spec().channels as usize;
- let bit_depth = reader.spec().bits_per_sample;
-
- let mut raw_data: Vec = Vec::new();
-
- // This is a very special case where the wav file holds a 64bit float spread over 4 16bit channels
- if num_channels == 4 && bit_depth == 16 {
- // Iterate over the samples and channels and push each sample to the vector
- let mut u64_holder: [u16; 4] = [0, 0, 0, 0];
- let mut current_channel: usize = 0;
-
- for sample in reader.samples::() {
- u64_holder[current_channel] = sample.unwrap() as u16;
- current_channel += 1;
- if current_channel == num_channels {
- raw_data.push(join_u16_into_f64(u64_holder));
- current_channel = 0;
- }
- }
- } else {
- for sample in reader.samples::() {
- raw_data.push(sample.unwrap() as f64);
- }
- }
-
- raw_data
-}
-
-fn join_u16_into_f64(bits: [u16; 4]) -> f64 {
- let u64_bits = (bits[0] as u64)
- | ((bits[1] as u64) << 16)
- | ((bits[2] as u64) << 32)
- | ((bits[3] as u64) << 48);
-
- f64::from_bits(u64_bits)
-}
-
-fn write_optimal_int_wav(filename: &str, data: Vec, bitdepth: i32, channels: i32) {
- let header: WavSpec = generate_wav_header(Some(channels), bitdepth as u16, 8000);
- let file_path = format!("{filename}.wav");
- let file = File::create(file_path).unwrap();
- let mut wav_writer = WavWriter::new(file, header).unwrap();
- for sample in data {
- let _ = wav_writer.write_sample(sample as i8);
- }
- let _ = wav_writer.finalize();
-}
-
-fn generate_wav_header(channels: Option, bitdepth: u16, samplerate: u32) -> WavSpec {
- hound::WavSpec {
- channels: channels.unwrap_or(4) as u16,
- sample_rate: samplerate,
- bits_per_sample: bitdepth,
- sample_format: hound::SampleFormat::Int,
- }
-}
-
-fn calculate_mid_channel(left: Vec, right: Vec) -> (Vec, Vec) {
- // We might have different sizes
- let min_size = left.len().min(right.len());
- let mut mid: Vec = Vec::with_capacity(min_size);
- let mut sides: Vec = Vec::with_capacity(min_size);
- // Formulas are easy, mid = 0.5*(left + right), sides = 0.5 * (left – right)
- for i in 0..min_size {
- mid.push((left[i] as i16 + right[i] as i16) / 2);
- sides.push((left[i] as i16 - right[i] as i16) / 2);
- }
- (mid, sides)
-}
-
-#[derive(Parser, Default, Debug)]
-struct Arguments {
- /// First wav file
- file_one: String,
- /// Second wav file
- file_two: String,
- /// reverse the process, picks the first file, writes in the second
- #[arg(long, action)]
- reverse: bool,
- /// Don't write a file, just dump the calculated contents
- #[arg(long, action)]
- dump: bool,
-}
-
-fn main() {
- let args = Arguments::parse();
- println!("{:?}", args);
-
- let binding_a: Vec = read_metrics_from_wav(&args.file_one);
- let binding_b: Vec = read_metrics_from_wav(&args.file_two);
- let (mid, sides) = calculate_mid_channel(binding_a, binding_b);
- if args.dump {
- println!("Mid: {:?}", mid);
- println!("Sides: {:?}", sides);
- } else {
- write_optimal_int_wav("mid", mid, 16, 1);
- write_optimal_int_wav("side", sides, 8, 1);
- }
-}
diff --git a/tools/src/bin/wav2wbro.rs b/tools/src/bin/wav2wbro.rs
index 7b784f7..7d9c7f2 100644
--- a/tools/src/bin/wav2wbro.rs
+++ b/tools/src/bin/wav2wbro.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use clap::{arg, command, Parser};
use log::debug;
use std::fs::File;
@@ -54,12 +70,11 @@ fn join_u16_into_f64(bits: [u16; 4]) -> f64 {
}
out
}
-// --- Legacy ends (I need to stop lying to myself...) ---
+// --- Legacy ends ---
#[derive(Parser, Default, Debug)]
#[command(author, version, about="WAV to WAVBRRO converter", long_about = None)]
struct Args {
- /// input file
input: PathBuf,
/// Verbose output, dumps everysample in the input file (for compression) and in the ouput file (for decompression)
@@ -75,16 +90,13 @@ fn main() {
assert!(is_wav_file(&arguments.input));
let wav_data = read_metrics_from_wav(filename);
let mut wb = WavBrro::new();
- // Clean NaN
wav_data.iter().for_each(|x| {
if !x.is_nan() {
wb.add_sample(*x)
}
});
- // Write the file
let wavbrro_file = format!("{}wbro", filename.strip_suffix("wav").unwrap());
wb.to_file(Path::new(&wavbrro_file));
- // Checking the results
if arguments.validate {
let brro_data = wb.get_samples();
assert_eq!(wav_data, brro_data);
diff --git a/vsri/src/lib.rs b/vsri/src/lib.rs
index 53529e6..0487e53 100644
--- a/vsri/src/lib.rs
+++ b/vsri/src/lib.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use chrono::{DateTime, Timelike, Utc};
use log::{debug, warn};
/// Very Small Rolo Index
@@ -56,18 +72,22 @@ pub fn start_day_ts(dt: DateTime) -> i64 {
/// # Examples
/// Creating a new index, metric is of expected time 0, but for sure location of X is 0
/// ```no_run
-/// let vsri = Vsri::new("metric_name", 0, 0);
+/// # use vsri::Vsri;
+/// let vsri = Vsri::new("metric_name");
/// vsri.flush();
/// ```
/// Updating an index, adding point at time 5sec
/// ```no_run
-/// let vsri = Vsri::load("metric_name").unwrap().update_for_point(5);
+///
+/// # use vsri::Vsri;
+/// let mut vsri = Vsri::load("metric_name").unwrap();
+/// vsri.update_for_point(5).unwrap();
/// vsri.flush();
/// ```
/// Fetch a sample location from the index given a timestamp
/// ```no_run
-/// let vsri = Vsri::load("metric_name").unwrap();
-/// vsri.get_sample_location("metric_name", 5);
+/// # use vsri::Vsri;
+/// let vsri = Vsri::get_sample_location("metric_name", 5);
/// ```
/// Index Structure
@@ -76,7 +96,7 @@ pub fn start_day_ts(dt: DateTime) -> i64 {
/// max_ts: the highest TS available in this file
/// vsri_segments: Description of each segment
/// [sample_rate (m), initial_point(x,y), # of samples(length)]
-/// Each segments describes a line with the form of mX + B that has a lenght
+/// Each segments describes a line with the form of mX + B that has a length
/// of # of samples.
#[derive(Debug, Default)]
pub struct Vsri {
@@ -90,7 +110,7 @@ pub struct Vsri {
impl Vsri {
/// Creates the index, it doesn't create the file in the disk
/// flush needs to be called for that
- pub fn new(filename: &String) -> Self {
+ pub fn new(filename: &str) -> Self {
debug!("[INDEX] Creating new index!");
Vsri {
index_file: filename.to_string(),
@@ -102,8 +122,8 @@ impl Vsri {
/// Given a filename and a time location, returns the sample location in the
/// data file. Or None in case it doesn't exist.
- pub fn get_sample_location(filename: String, y: i32) -> Option {
- let vsri = match Vsri::load(&filename) {
+ pub fn get_sample_location(filename: &str, y: i32) -> Option {
+ let vsri = match Vsri::load(filename) {
Ok(vsri) => vsri,
Err(_err) => return None,
};
diff --git a/wavbrro/src/lib.rs b/wavbrro/src/lib.rs
index ceca829..3f1d728 100644
--- a/wavbrro/src/lib.rs
+++ b/wavbrro/src/lib.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
pub mod read;
pub mod wavbrro;
pub mod write;
diff --git a/wavbrro/src/read.rs b/wavbrro/src/read.rs
index 449de01..000379e 100644
--- a/wavbrro/src/read.rs
+++ b/wavbrro/src/read.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use std::fs;
use std::fs::File;
use std::io::{self, Read, Seek, SeekFrom};
diff --git a/wavbrro/src/wavbrro.rs b/wavbrro/src/wavbrro.rs
index a6caeed..651a910 100644
--- a/wavbrro/src/wavbrro.rs
+++ b/wavbrro/src/wavbrro.rs
@@ -1,3 +1,19 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use rkyv::{Archive, Deserialize, Serialize};
use std::path::Path;
use std::{error, fmt, io, result};
@@ -76,7 +92,7 @@ impl WavBrro {
self.sample_count += 1;
}
- // This should be generic, but first implementation is going to be Vec f64
+ // TODO: This should be generic, but first implementation is going to be Vec f64
// This consumes self!
pub fn get_samples(self) -> Vec {
self.chunks.into_iter().flatten().collect::>()
diff --git a/wavbrro/src/write.rs b/wavbrro/src/write.rs
index ab284c7..e21cd25 100644
--- a/wavbrro/src/write.rs
+++ b/wavbrro/src/write.rs
@@ -1,11 +1,25 @@
+/*
+Copyright 2024 NetApp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
use std::fs::File;
use std::os::unix::prelude::FileExt;
use std::path::Path;
pub fn write_wavbrro_file(file_path: &Path, content: &[u8]) {
- // The content of the header
let header: [u8; 12] = *b"WBRO0000WBRO";
- // We need to put the header in front
let file = File::create(file_path).expect("Can't create file!");
file.write_at(&header, 0).expect("Fail to write header");
file.write_at(content, header.len() as u64)