From a26bc86ccc9cae09d774c2df1ea3a3dd0f99c894 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Wed, 22 Jan 2025 17:08:24 -0500 Subject: [PATCH] Remove typing imports for List, Tuple, Union --- src/hats_import/catalog/arguments.py | 7 +++---- src/hats_import/catalog/resume_plan.py | 13 ++++++------- src/hats_import/index/arguments.py | 7 +++---- .../margin_cache/margin_cache_arguments.py | 3 +-- .../margin_cache/margin_cache_resume_plan.py | 5 ++--- src/hats_import/soap/map_reduce.py | 4 +--- src/hats_import/soap/resume_plan.py | 7 +++---- 7 files changed, 19 insertions(+), 27 deletions(-) diff --git a/src/hats_import/catalog/arguments.py b/src/hats_import/catalog/arguments.py index 6bc13535..2d95d482 100644 --- a/src/hats_import/catalog/arguments.py +++ b/src/hats_import/catalog/arguments.py @@ -4,7 +4,6 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import List from hats.catalog import TableProperties from hats.pixel_math import spatial_index @@ -24,9 +23,9 @@ class ImportArguments(RuntimeArguments): """level of catalog data, object (things in the sky) or source (detections)""" input_path: str | Path | UPath | None = None """path to search for the input data""" - input_file_list: List[str | Path | UPath] = field(default_factory=list) + input_file_list: list[str | Path | UPath] = field(default_factory=list) """can be used instead of input_path to import only specified files""" - input_paths: List[str | Path | UPath] = field(default_factory=list) + input_paths: list[str | Path | UPath] = field(default_factory=list) """resolved list of all files that will be used in the importer""" ra_column: str = "ra" @@ -71,7 +70,7 @@ class ImportArguments(RuntimeArguments): """healpix order to use when mapping. will be ``highest_healpix_order`` unless a positive value is provided for ``constant_healpix_order``""" - run_stages: List[str] = field(default_factory=list) + run_stages: list[str] = field(default_factory=list) """list of parallel stages to run. options are ['mapping', 'splitting', 'reducing', 'finishing']. ['planning', 'binning'] stages are not optional. this can be used to force the pipeline to stop after an early stage, to allow the diff --git a/src/hats_import/catalog/resume_plan.py b/src/hats_import/catalog/resume_plan.py index 5cec0e01..13c9816c 100644 --- a/src/hats_import/catalog/resume_plan.py +++ b/src/hats_import/catalog/resume_plan.py @@ -5,7 +5,6 @@ import pickle import re from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple import hats.pixel_math.healpix_shim as hp import numpy as np @@ -23,13 +22,13 @@ class ResumePlan(PipelineResumePlan): """Container class for holding the state of each file in the pipeline plan.""" - input_paths: List[UPath] = field(default_factory=list) + input_paths: list[UPath] = field(default_factory=list) """resolved list of all files that will be used in the importer""" - map_files: List[Tuple[str, str]] = field(default_factory=list) + map_files: list[tuple[str, str]] = field(default_factory=list) """list of files (and job keys) that have yet to be mapped""" - split_keys: List[Tuple[str, str]] = field(default_factory=list) + split_keys: list[tuple[str, str]] = field(default_factory=list) """set of files (and job keys) that have yet to be split""" - destination_pixel_map: Optional[Dict[HealpixPixel, int]] = None + destination_pixel_map: dict[HealpixPixel, int] | None = None """Destination pixels and their expected final count""" should_run_mapping: bool = True should_run_splitting: bool = True @@ -55,7 +54,7 @@ def __init__( tmp_base_path: UPath | None = None, delete_resume_log_files: bool = True, delete_intermediate_parquet_files: bool = True, - run_stages: List[str] | None = None, + run_stages: list[str] | None = None, import_args=None, ): if import_args: @@ -76,7 +75,7 @@ def __init__( self.input_paths = input_paths self.gather_plan(run_stages) - def gather_plan(self, run_stages: List[str] | None = None): + def gather_plan(self, run_stages: list[str] | None = None): """Initialize the plan.""" with self.print_progress(total=4, stage_name="Planning") as step_progress: ## Make sure it's safe to use existing resume state. diff --git a/src/hats_import/index/arguments.py b/src/hats_import/index/arguments.py index b3fc9c7e..98b80616 100644 --- a/src/hats_import/index/arguments.py +++ b/src/hats_import/index/arguments.py @@ -4,7 +4,6 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import List, Optional from hats import read_hats from hats.catalog import Catalog, TableProperties @@ -20,9 +19,9 @@ class IndexArguments(RuntimeArguments): ## Input input_catalog_path: str | Path | UPath | None = None - input_catalog: Optional[Catalog] = None + input_catalog: Catalog | None = None indexing_column: str = "" - extra_columns: List[str] = field(default_factory=list) + extra_columns: list[str] = field(default_factory=list) ## Output include_healpix_29: bool = True @@ -40,7 +39,7 @@ class IndexArguments(RuntimeArguments): ## Compute parameters compute_partition_size: int = 1_000_000_000 """partition size used when creating leaf parquet files.""" - division_hints: Optional[List] = None + division_hints: list | None = None """Hints used when splitting up the rows by the new index. If you have some prior knowledge about the distribution of your indexing_column, providing it here can speed up calculations dramatically. Note that diff --git a/src/hats_import/margin_cache/margin_cache_arguments.py b/src/hats_import/margin_cache/margin_cache_arguments.py index b56a2a4e..ba6d23b3 100644 --- a/src/hats_import/margin_cache/margin_cache_arguments.py +++ b/src/hats_import/margin_cache/margin_cache_arguments.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import List import hats.pixel_math.healpix_shim as hp from hats import read_hats @@ -34,7 +33,7 @@ class MarginCacheArguments(RuntimeArguments): input_catalog_path: str | Path | UPath | None = None """the path to the hats-formatted input catalog.""" - debug_filter_pixel_list: List[HealpixPixel] = field(default_factory=list) + debug_filter_pixel_list: list[HealpixPixel] = field(default_factory=list) """debug setting. if provided, we will first filter the catalog to the pixels provided. this can be useful for creating a margin over a subset of a catalog.""" diff --git a/src/hats_import/margin_cache/margin_cache_resume_plan.py b/src/hats_import/margin_cache/margin_cache_resume_plan.py index 030bd920..d93a48a7 100644 --- a/src/hats_import/margin_cache/margin_cache_resume_plan.py +++ b/src/hats_import/margin_cache/margin_cache_resume_plan.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import List import pandas as pd from hats import pixel_math @@ -19,8 +18,8 @@ class MarginCachePlan(PipelineResumePlan): """Container class for holding the state of each file in the pipeline plan.""" margin_pair_file: str | None = None - partition_pixels: List[HealpixPixel] = field(default_factory=list) - combined_pixels: List[HealpixPixel] = field(default_factory=list) + partition_pixels: list[HealpixPixel] = field(default_factory=list) + combined_pixels: list[HealpixPixel] = field(default_factory=list) MAPPING_STAGE = "mapping" REDUCING_STAGE = "reducing" diff --git a/src/hats_import/soap/map_reduce.py b/src/hats_import/soap/map_reduce.py index 44d8c612..e78f1565 100644 --- a/src/hats_import/soap/map_reduce.py +++ b/src/hats_import/soap/map_reduce.py @@ -1,7 +1,5 @@ """Inner methods for SOAP""" -from typing import List - import numpy as np import pandas as pd import pyarrow.parquet as pq @@ -79,7 +77,7 @@ def _write_count_results(cache_path, source_healpix, results): ) -def count_joins(soap_args: SoapArguments, source_pixel: HealpixPixel, object_pixels: List[HealpixPixel]): +def count_joins(soap_args: SoapArguments, source_pixel: HealpixPixel, object_pixels: list[HealpixPixel]): """Count the number of equijoined sources in the object pixels. If any un-joined source pixels remain, stretch out to neighboring object pixels. diff --git a/src/hats_import/soap/resume_plan.py b/src/hats_import/soap/resume_plan.py index 93a3d710..80313d87 100644 --- a/src/hats_import/soap/resume_plan.py +++ b/src/hats_import/soap/resume_plan.py @@ -4,7 +4,6 @@ import re from dataclasses import dataclass, field -from typing import List, Optional, Tuple import hats.pixel_math.healpix_shim as hp import numpy as np @@ -22,11 +21,11 @@ class SoapPlan(PipelineResumePlan): """Container class for holding the state of each file in the pipeline plan.""" - count_keys: List[Tuple[HealpixPixel, List[HealpixPixel], str]] = field(default_factory=list) + count_keys: list[tuple[HealpixPixel, list[HealpixPixel], str]] = field(default_factory=list) """set of pixels (and job keys) that have yet to be counted""" - reduce_keys: List[Tuple[HealpixPixel, str]] = field(default_factory=list) + reduce_keys: list[tuple[HealpixPixel, str]] = field(default_factory=list) """set of object catalog pixels (and job keys) that have yet to be reduced/combined""" - source_pixel_map: Optional[List[Tuple[HealpixPixel, List[HealpixPixel], str]]] = None + source_pixel_map: list[tuple[HealpixPixel, list[HealpixPixel], str]] | None = None """Map of object pixels to source pixels, with counting key.""" object_catalog: Catalog | None = None