Skip to content

Commit

Permalink
docs: Refactor docs directory hierarchy
Browse files Browse the repository at this point in the history
Split docs folder into two subfolders.
All the main docs content lives in `docs/docs`.
Dynamically generated content lives in `docs/docs-res`.
This split prevents the MkDocs live reloading from entering an infinite loop
when generating data files, images, or pages, for the documentation.
  • Loading branch information
rodrigogiraoserrao committed Sep 20, 2024
1 parent b7e5e09 commit cd12eec
Show file tree
Hide file tree
Showing 200 changed files with 180 additions and 161 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/docs-global.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
ref: ${{ github.event.client_payload.sha }}
- uses: gaurav-nelson/github-action-markdown-link-check@v1
with:
config-file: docs/mlc-config.json
config-file: docs/docs/mlc-config.json
folder-path: docs

lint:
Expand All @@ -39,13 +39,13 @@ jobs:
- uses: chartboost/ruff-action@v1
with:
src: docs/
src: docs/docs/
version: ${{ steps.version.outputs.version }}
args: check --no-fix

- uses: chartboost/ruff-action@v1
with:
src: docs/
src: docs/docs/
version: ${{ steps.version.outputs.version }}
args: format --diff

Expand All @@ -69,7 +69,7 @@ jobs:
echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
- name: Install Python dependencies
run: uv pip install -r py-polars/requirements-dev.txt -r docs/requirements.txt
run: uv pip install -r py-polars/requirements-dev.txt -r docs/docs/requirements.txt

- name: Set up Rust
run: rustup show
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/lint-rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
pull_request:
paths:
- crates/**
- docs/src/rust/**
- docs/docs/src/rust/**
- examples/**
- py-polars/src/**
- py-polars/Cargo.toml
Expand All @@ -15,7 +15,7 @@ on:
- main
paths:
- crates/**
- docs/src/rust/**
- docs/docs/src/rust/**
- examples/**
- py-polars/src/**
- py-polars/Cargo.toml
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/test-python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
paths:
- Cargo.lock
- py-polars/**
- docs/src/python/**
- docs/docs/src/python/**
- crates/**
- .github/workflows/test-python.yml
push:
Expand All @@ -14,7 +14,7 @@ on:
paths:
- Cargo.lock
- crates/**
- docs/src/python/**
- docs/docs/src/python/**
- py-polars/**
- .github/workflows/test-python.yml

Expand Down
5 changes: 2 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,5 @@ target/
*.tbl

# Project
/docs/data/
/docs/images/
/docs/people.md
/docs/docs-res/data/
/docs/docs-res/people.md
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
resolver = "2"
members = [
"crates/*",
"docs/src/rust",
"docs/docs/src/rust",
# "examples/*",
"py-polars",
]
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ requirements: .venv ## Install/refresh Python project requirements
-r py-polars/requirements-dev.txt \
-r py-polars/requirements-lint.txt \
-r py-polars/docs/requirements-docs.txt \
-r docs/requirements.txt
-r docs/docs/requirements.txt

.PHONY: requirements-all
requirements-all: .venv ## Install/refresh all Python requirements (including those needed for CI tests)
Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ shape: (5, 8)
## SQL

```python
>>> df = pl.scan_csv("docs/data/iris.csv")
>>> df = pl.scan_csv("docs/docs-res/data/iris.csv")
>>> ## OPTION 1
>>> # run SQL queries on frame-level
>>> df.sql("""
Expand Down Expand Up @@ -141,14 +141,14 @@ SQL commands can also be run directly from your terminal using the Polars CLI:

```bash
# run an inline SQL query
> polars -c "SELECT species, AVG(sepal_length) AS avg_sepal_length, AVG(sepal_width) AS avg_sepal_width FROM read_csv('docs/data/iris.csv') GROUP BY species;"
> polars -c "SELECT species, AVG(sepal_length) AS avg_sepal_length, AVG(sepal_width) AS avg_sepal_width FROM read_csv('docs/docs-res/data/iris.csv') GROUP BY species;"

# run interactively
> polars
Polars CLI v0.3.0
Type .help for help.

> SELECT species, AVG(sepal_length) AS avg_sepal_length, AVG(sepal_width) AS avg_sepal_width FROM read_csv('docs/data/iris.csv') GROUP BY species;
> SELECT species, AVG(sepal_length) AS avg_sepal_length, AVG(sepal_width) AS avg_sepal_width FROM read_csv('docs/docs-res/data/iris.csv') GROUP BY species;
```

Refer to the [Polars CLI repository](https://github.com/pola-rs/polars-cli) for more information.
Expand Down
4 changes: 2 additions & 2 deletions crates/polars/tests/it/io/parquet/arrow/read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ fn all_types() -> PolarsResult<()> {
use crate::io::parquet::read::file::FileReader;

let dir = env!("CARGO_MANIFEST_DIR");
let path = PathBuf::from(dir).join("../../docs/data/alltypes_plain.parquet");
let path = PathBuf::from(dir).join("../../docs/docs-res/data/alltypes_plain.parquet");

let mut reader = std::fs::File::open(path)?;

Expand Down Expand Up @@ -55,7 +55,7 @@ fn all_types_chunked() -> PolarsResult<()> {

use crate::io::parquet::read::file::FileReader;
let dir = env!("CARGO_MANIFEST_DIR");
let path = PathBuf::from(dir).join("../../docs/data/alltypes_plain.parquet");
let path = PathBuf::from(dir).join("../../docs/docs-res/data/alltypes_plain.parquet");
let mut reader = std::fs::File::open(path)?;

let metadata = read_metadata(&mut reader)?;
Expand Down
2 changes: 1 addition & 1 deletion crates/polars/tests/it/io/parquet/read/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub fn hybrid_rle_iter(d: HybridRleDecoder) -> ParquetResult<std::vec::IntoIter<

pub fn get_path() -> PathBuf {
let dir = env!("CARGO_MANIFEST_DIR");
PathBuf::from(dir).join("../../docs/data")
PathBuf::from(dir).join("../../docs/docs-res/data")
}

/// Reads a page into an [`Array`].
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 2 additions & 0 deletions docs/docs-res/images/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*
!.gitignore
File renamed without changes.
File renamed without changes
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
)

# Load all links to reference docs
with open("docs/_build/API_REFERENCE_LINKS.yml", "r") as f:
with open("docs/docs/_build//API_REFERENCE_LINKS.yml", "r") as f:
API_REFERENCE_LINKS = yaml.load(f, Loader=yaml.CLoader)


Expand All @@ -39,7 +39,7 @@ def create_feature_flag_link(feature_name: str) -> str:


def create_feature_flag_links(language: str, api_functions: List[str]) -> List[str]:
"""Generate markdown feature flags for the code tas based on the api_functions.
"""Generate markdown feature flags for the code tabs based on the api_functions.
It checks for the key feature_flag in the configuration yaml for the function and if it exists print out markdown
Args:
Expand Down Expand Up @@ -165,7 +165,7 @@ def code_block(

for language, info in LANGUAGES.items():
base_path = f"{language}/{path}{info['extension']}"
full_path = "docs/src/" + base_path
full_path = "docs/docs/src/" + base_path
# Check if file exists for the language
if os.path.exists(full_path):
result.append(code_tab(base_path, section, info, api_functions))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@

ICON_TEMPLATE = '<a href="{html_url}"><img alt="{login}" class="contributor_icon" src="{avatar_url}&s=40" loading="lazy" /></a>'


def get_people_md():
repo = g.get_repo("pola-rs/polars")
contributors = repo.get_contributors()
with open("./docs/people.md", "w") as f:
with open("./docs/docs-res/people.md", "w") as f:
for c in itertools.islice(contributors, 50):
# We love dependabot, but he doesn't need a spot on our website
if c.login == "dependabot[bot]":
Expand All @@ -28,11 +29,11 @@ def get_people_md():


def on_startup(command, dirty):
"""Mkdocs hook to autogenerate docs/people.md on startup"""
"""Mkdocs hook to autogenerate docs/docs/people.md on startup"""
try:
get_people_md()
except Exception as e:
msg = f"WARNING:{__file__}: Could not generate docs/people.md. Got error: {str(e)}"
msg = f"WARNING:{__file__}: Could not generate docs/docs/people.md. Got error: {str(e)}"
print(msg)


Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ This guide shows how to set up a debugging environment that makes debugging Rust
Start by installing the CodeLLDB extension (see above).
Then add the following two configurations to your `launch.json` file.
This file is usually found in the `.vscode` folder of your project root.
See the [official VSCode documentation](https://code.visualstudio.com/docs/editor/debugging#_launch-configurations) for more information about the `launch.json` file.
See the [official VSCode documentation](https://code.visualstudio.com/docs/docs/editor/debugging#_launch-configurations) for more information about the `launch.json` file.

<details><summary><code><b>launch.json</b></code></summary>

Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion docs/index.md → docs/docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ A more extensive introduction can be found in the [next chapter](user-guide/gett

Polars has a very active community with frequent releases (approximately weekly). Below are some of the top contributors to the project:

--8<-- "docs/people.md"
--8<-- "docs/docs-res/people.md"

## Contributing

Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import polars as pl

q = (
pl.scan_csv("docs/data/iris.csv")
pl.scan_csv("docs/docs-res/data/iris.csv")
.filter(pl.col("sepal_length") > 5)
.group_by("species")
.agg(pl.all().sum())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@

# --8<-- [start:eager]

df = pl.read_csv("docs/data/iris.csv")
df = pl.read_csv("docs/docs-res/data/iris.csv")
df_small = df.filter(pl.col("sepal_length") > 5)
df_agg = df_small.group_by("species").agg(pl.col("sepal_width").mean())
print(df_agg)
# --8<-- [end:eager]

# --8<-- [start:lazy]
q = (
pl.scan_csv("docs/data/iris.csv")
pl.scan_csv("docs/docs-res/data/iris.csv")
.filter(pl.col("sepal_length") > 5)
.group_by("species")
.agg(pl.col("sepal_width").mean())
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
# --8<-- [start:import]
import polars as pl

# --8<-- [end:import]

# --8<-- [start:streaming]
q1 = (
pl.scan_csv("docs/data/iris.csv")
pl.scan_csv("docs/docs-res/data/iris.csv")
.filter(pl.col("sepal_length") > 5)
.group_by("species")
.agg(pl.col("sepal_width").mean())
Expand All @@ -18,7 +19,7 @@
# --8<-- [end:example]

# --8<-- [start:example2]
q2 = pl.scan_csv("docs/data/iris.csv").with_columns(
q2 = pl.scan_csv("docs/docs-res/data/iris.csv").with_columns(
pl.col("sepal_length").mean().over("species")
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,24 @@
# --8<-- [end:dataframe]

# --8<-- [start:csv]
df.write_csv("docs/data/output.csv")
df_csv = pl.read_csv("docs/data/output.csv")
df.write_csv("docs/docs-res/data/output.csv")
df_csv = pl.read_csv("docs/docs-res/data/output.csv")
print(df_csv)
# --8<-- [end:csv]

# --8<-- [start:csv2]
df_csv = pl.read_csv("docs/data/output.csv", try_parse_dates=True)
df_csv = pl.read_csv("docs/docs-res/data/output.csv", try_parse_dates=True)
print(df_csv)
# --8<-- [end:csv2]

# --8<-- [start:json]
df.write_json("docs/data/output.json")
df_json = pl.read_json("docs/data/output.json")
df.write_json("docs/docs-res/data/output.json")
df_json = pl.read_json("docs/docs-res/data/output.json")
print(df_json)
# --8<-- [end:json]

# --8<-- [start:parquet]
df.write_parquet("docs/data/output.parquet")
df_parquet = pl.read_parquet("docs/data/output.parquet")
df.write_parquet("docs/docs-res/data/output.parquet")
df_parquet = pl.read_parquet("docs/docs-res/data/output.parquet")
print(df_parquet)
# --8<-- [end:parquet]
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@

"""
# --8<-- [start:read]
df = pl.read_csv("docs/data/path.csv")
df = pl.read_csv("docs/docs-res/data/path.csv")
# --8<-- [end:read]
"""

# --8<-- [start:write]
df = pl.DataFrame({"foo": [1, 2, 3], "bar": [None, "bak", "baz"]})
df.write_csv("docs/data/path.csv")
df.write_csv("docs/docs-res/data/path.csv")
# --8<-- [end:write]

# --8<-- [start:scan]
df = pl.scan_csv("docs/data/path.csv")
df = pl.scan_csv("docs/docs-res/data/path.csv")
# --8<-- [end:scan]
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,22 @@

"""
# --8<-- [start:read]
df = pl.read_excel("docs/data/path.xlsx")
df = pl.read_excel("docs/docs-res/data/path.xlsx")
# --8<-- [end:read]
# --8<-- [start:read_sheet_name]
df = pl.read_excel("docs/data/path.xlsx", sheet_name="Sales")
df = pl.read_excel("docs/docs-res/data/path.xlsx", sheet_name="Sales")
# --8<-- [end:read_sheet_name]
"""

# --8<-- [start:write]
df = pl.DataFrame({"foo": [1, 2, 3], "bar": [None, "bak", "baz"]})
df.write_excel("docs/data/path.xlsx")
df.write_excel("docs/docs-res/data/path.xlsx")
# --8<-- [end:write]

"""
# --8<-- [start:write_sheet_name]
df = pl.DataFrame({"foo": [1, 2, 3], "bar": [None, "bak", "baz"]})
df.write_excel("docs/data/path.xlsx", worksheet="Sales")
df.write_excel("docs/docs-res/data/path.xlsx", worksheet="Sales")
# --8<-- [end:write_sheet_name]
"""
Loading

0 comments on commit cd12eec

Please sign in to comment.