From 8e792d261b9e91226918786a5555f35e4007ce7d Mon Sep 17 00:00:00 2001 From: RobH Date: Mon, 21 Oct 2024 22:16:58 +0000 Subject: [PATCH 01/18] Setup new file for hand 4.5.11.1 data load --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 2758 +++++++++++++++++ 1 file changed, 2758 insertions(+) create mode 100644 Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb new file mode 100644 index 00000000..de767307 --- /dev/null +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -0,0 +1,2758 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f15f66a5-1f40-4933-8601-b56e2e8134a9", + "metadata": {}, + "source": [ + "### Notes - Oct 21, 2024 ###\n", + "This is a copy from 10.FIM Version 4.5.2.11 which included hand data loads plus ras2fim data. \n", + "We will remove all ras2fim stuff here knowing that sometimes ras2fim will be uploaded on its own.\n", + "However.. when ras2fim is loaded, some steps here will need to be re-run. Those steps will be\n", + "duplicated when we build our next ras2fim load. This hand release does not have a ras2fim update so\n", + "we will keep the one in place.\n", + "

\n", + "All code in here will be reviewed and adjusted as the loads progress. Consider each step to be\n", + "WIP until you see a load date below.\n" + ] + }, + { + "cell_type": "markdown", + "id": "eae54700-98da-48c9-95d3-97dde219b8d5", + "metadata": { + "tags": [] + }, + "source": [ + "### Load Status for hand 4.5.11.1 - Started Oct 21, 2024\n", + "\n", + "#### Add dates to each line as they have been loaded\n", + "\n", + "1. `Crosswalk` : \n", + "2. `Lambda FIM_PREFIX` : \n", + "3. `Lambda FIM_VERSION and Memory` : \n", + "4. `ras2fim` : No update in this release. Keep the last load.\n", + "5. `ras2fim Boundaries`: No update required for this release.\n", + "6. `AEP`\n", + " - `2 year` : -- \n", + " - `5 year` : -- \n", + " - `10 year` : -- \n", + " - `25 year` : -- \n", + " - `50 year` : -- \n", + " - `HW / High Water` : -- \n", + " - `Change the hv-vpp-ti-viz-fim-data-prep Lambda memory back to 2048mb` : -- \n", + "7. `Catchments`\n", + " - `Branch 0` : -- \n", + " - `GMS` : -- \n", + "8. `usgs_elev_table` : -- \n", + "9. `hydrotable / hydrotable_staggered` : -- \n", + "10. `usgs_rating_curve / usgs_rating_curves staggered` : -- \n", + "11. `Skills Metrics` : -- \n", + "12. `FIM Performance` : -- \n", + "13. `CatFIM`\n", + " - `Stage Based CatFIM` : -- \n", + " - `Flow Based CatFIM` : -- \n", + " - `CatFIM FIM 30` : Stage based only? flow not needed but confirm this.\n", + "14. `Clear HAND cache` :\n", + "15. `GIT1 and `Terraform ??` : TBD\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e1ee2b5-b109-49e8-8255-f06449b44ee6", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "# Cell to manually pip reload a packages that the Jupyter engine not retained\n", + "# !pip install numpy\n", + "# !pip install geopandas\n", + "# !pip install pyarrow\n", + "# !pip install xarray\n", + "# !pip install geoalchemy2\n", + "# !pip install contextily\n", + "# !pip install rioxarray\n", + "print(\"All loaded\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "103a0d12-b2a0-4586-ba0f-0ed6b2c4eb89", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# pd.set_option(\"max_info_rows\", 100000) # override " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b656259", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import codecs\n", + "import csv\n", + "import sys\n", + "\n", + "from datetime import datetime\n", + "\n", + "import boto3\n", + "import geopandas as gpd\n", + "import json\n", + "import pandas as pd\n", + "import s3fs\n", + "import sqlalchemy\n", + "import xarray as xr\n", + "\n", + "from geopandas import GeoDataFrame\n", + "from io import StringIO\n", + "from geoalchemy2 import Geometry\n", + "from shapely import wkt\n", + "from shapely.geometry import Polygon\n", + "from sqlalchemy.exc import DataError # yes, reduntant, fix it later\n", + "from sqlalchemy.types import Text # yes, reduntant, fix it later\n", + "\n", + "sys.path.append(os.path.join(os.path.abspath(''), '..'))\n", + "\n", + "import helper_functions.s3_shared_functions as s3_sf\n", + "import helper_functions.shared_functions as sf\n", + "# import helper_functions.viz_classes\n", + "\n", + "from helper_functions.viz_classes import database\n", + "from helper_functions.viz_db_ingest import lambda_handler as execute_db_ingest\n", + "\n", + "print(\"imports loaded\")\n", + "\n", + "# Note: Aug 2024: Sometimes if you need to do the pip install above, you need to reload this twice.. must be a circular dependency ?? or forced pkg reload\n", + "# Not sure why yet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e944eff3-6023-48f4-9f57-27304a240447", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Aug 5, 2024: # The variable named FIM_VERSION will continue to be the field that joins all data together.\n", + "# But we need two new public display fields in the UI. We will no longer show a UI field which previously would have been\n", + "# \"FIM_4_5_2_11\". That won't be displayed anymore. The public field of:\n", + "# public_fim_version, for this edition, becomes \"FIM 5_0_0\" (yes.. three segs)\n", + "# public_model_version, for this edition, becomes \"HAND 4_5_2_11\". \n", + "# When we add ras2fim into the system, it's public_fim_version continues to be FIM 5_0_0,\n", + "# but ras2fim public_model_version, becomes \"ras2fim 2_0_3_0\"\n", + "\n", + "# NOTE: sep 19, 2024: creaing the 4.4.0.0 was just for learning purposes. Now Rob has access to UAT db's so I can compare against\n", + "# those next time if needed. We might remove the references to 4.4.0.0 next time\n", + "\n", + "OLD_FIM_VERSION = \"4.4.0.0\"\n", + "NEW_FIM_VERSION = \"4.5.2.11\"\n", + "PUBLIC_FIM_VERSION = \"FIM 5.0.0\" \n", + "FIM_MODEL_VERSION = \"HAND 4.5.2.11\" # on next major build (after Aug 2024, change this to space and dots. ie) HAND 4.5.2.11)\n", + "OLD_FIM_TAG = OLD_FIM_VERSION.replace('.', '_')\n", + "\n", + "FIM_ROOT_DPATH = f\"fim/fim_{NEW_FIM_VERSION.replace('.', '_')}\"\n", + "HAND_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/hand_datasets\"\n", + "QA_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/qa_datasets\"\n", + "\n", + "FIM_BUCKET = \"hydrovis-ti-deployment-us-east-1\"\n", + "FIM_CROSSWALK_FPATH = os.path.join(HAND_DATASETS_DPATH, \"crosswalk_table.csv\")\n", + "PIPELINE_ARN = 'arn:aws:states:us-east-1:526904826677:stateMachine:hv-vpp-ti-viz-pipeline'\n", + "\n", + "COLUMN_NAME_FIM_VERSION = \"fim_version\"\n", + "COLUMN_NAME_MODEL_VERSION = \"model_version\"\n", + "\n", + "# Sometimes these credential values get updated. To find the latest correct values, go to your AWS Console log page and click on the \"Access Key\"\n", + "# link to get the latest valid set. Using the \"AWS environment variables\" values.\n", + "# If this is not set correctly, you will get an HTTP error 400 when you call S3 lower.\n", + "# You might also see an error of 'An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist.\" the creds are not correct\"\n", + "\n", + "# Helps us get to the keys. Note: This was added Oct 16, 2024 and is untested\n", + "sys.path.append(os.path.join(os.path.abspath(''), '../../../../AWS_Secret_keys'))\n", + "import AWS_Keys\n", + "\n", + "\n", + "S3_CLIENT = boto3.client(\"s3\")\n", + "STEPFUNCTION_CLIENT = boto3.client('stepfunctions')\n", + "VIZ_DB_ENGINE = sf.get_db_engine('viz')\n", + "\n", + "print(\"Global Variables loaded\")" + ] + }, + { + "cell_type": "markdown", + "id": "6a5827eb-3ca3-4b29-8ee7-f7c4f3c2c736", + "metadata": { + "tags": [] + }, + "source": [ + "

1 - UPLOAD FIM4 HYDRO ID/FEATURE ID CROSSWALK

" + ] + }, + { + "cell_type": "markdown", + "id": "28195858-0cb6-4ad3-8966-db1371a8452a", + "metadata": { + "tags": [] + }, + "source": [ + "February 2024 Update from Tyler: This code will need to be updated to handle a new hand_id unique integer that the fim team (Rob Hanna and Matt Luck) has added to the crosswalk, and is now important to fim runs. They also changed the field names / format to match our schema, so this chunk of code should be able to be simplified significantly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "007c59f4-72ff-4e3e-9f99-36a8e830c426", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + " # This was already done for 4.4.0.0, so we can skip it jump from 4.5.2.0 to 4.2.5.11\n", + " \n", + "\n", + "'''\n", + "Be Very Careful to just rename tables. If they have indexes, the index will now point to the new\n", + "table names but maintain the original index name. Those index names can really mess stuff up.\n", + "Best to never rename unless you rename indexes as well. This particular on is ok. \n", + "Note: When various '\"to_sql\" tools are run which have GIST indexes, this index column name issue\n", + "will be the problem.\n", + "\n", + "Why Drop instead of Truncate? if the schema changes for the incoming, truncate will have column\n", + "missmatches.\n", + "\n", + "We really should be backing up indexes and constraints as well.\n", + "\n", + "'''\n", + "\n", + "\n", + "# TODO: Aug 2024: Change this to a backup without indexes and not rename, it affects indexes that we might need\n", + "# tbl_name = \"derived.fim4_featureid_crosswalk\"\n", + "# new_table_name = f\"{tbl_name}_{OLD_FIM_TAG}\"\n", + "# sql = f'''\n", + "# CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE {tbl_name};\n", + "# '''\n", + "# sf.execute_sql(sql)\n", + "# print(f\"{tbl_name} copied to {new_table_name} if it does not already exists\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c96a49f2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "print(f\"Getting column name from {FIM_CROSSWALK_FPATH}\")\n", + "\n", + "data = S3_CLIENT.get_object(Bucket=FIM_BUCKET, Key=FIM_CROSSWALK_FPATH)\n", + "d_reader = csv.DictReader(codecs.getreader(\"utf-8\")(data[\"Body\"]))\n", + "headers = d_reader.fieldnames\n", + "\n", + "# Aug 5, 2024 - Updated column names for 4.5.2.11\n", + "header_str = \"(\"\n", + "for header in headers:\n", + " header_str += header\n", + " if header in ['hand_id', 'hydro_id', 'lake_id']:\n", + " header_str += ' integer,'\n", + " elif header in ['branch_id', 'feature_id']:\n", + " header_str += ' bigint,'\n", + " else:\n", + " header_str += ' TEXT,'\n", + "header_str = header_str[:-1] + \")\"\n", + "print(header_str)\n", + "\n", + "db = database(db_type=\"viz\")\n", + "with db.get_db_connection() as conn, conn.cursor() as cur:\n", + " \n", + " print(f\"Deleting/Creating derived.fim4_featureid_crosswalk using columns {header_str}\")\n", + " sql = f\"DROP TABLE IF EXISTS derived.fim4_featureid_crosswalk; CREATE TABLE derived.fim4_featureid_crosswalk {header_str};\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + " print(f\"Importing {FIM_CROSSWALK_FPATH} to derived.fim4_featureid_crosswalk\")\n", + " sql = f\"\"\"\n", + " SELECT aws_s3.table_import_from_s3(\n", + " 'derived.fim4_featureid_crosswalk',\n", + " '', \n", + " '(format csv, HEADER true)',\n", + " (SELECT aws_commons.create_s3_uri(\n", + " '{FIM_BUCKET}',\n", + " '{FIM_CROSSWALK_FPATH}',\n", + " 'us-east-1'\n", + " ) AS s3_uri\n", + " ),\n", + " aws_commons.create_aws_credentials('{TI_ACCESS_KEY}', '{TI_SECRET_KEY}', '{TI_TOKEN}')\n", + " );\n", + " \"\"\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + " \n", + " # Aug 5, 2024: see notes at the top about the new FIM 5.0.0 system \n", + " # We will manually add a couple of new columns for public display\n", + " # New columns names are public_fim_version (FIM_5_0_0) and public model version (FIM_4_5_2_11)\n", + " print(f\"Adding {COLUMN_NAME_FIM_VERSION} column to derived.fim4_featureid_crosswalk\")\n", + " sql = f\"ALTER TABLE derived.fim4_featureid_crosswalk ADD COLUMN IF NOT EXISTS {COLUMN_NAME_FIM_VERSION} text DEFAULT '{PUBLIC_FIM_VERSION}';\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + " \n", + " print(f\"Adding {COLUMN_NAME_MODEL_VERSION} column to derived.fim4_featureid_crosswalk\")\n", + " sql = f\"ALTER TABLE derived.fim4_featureid_crosswalk ADD COLUMN IF NOT EXISTS {COLUMN_NAME_MODEL_VERSION} text DEFAULT '{FIM_MODEL_VERSION}';\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + " print(\"Adding feature id index to derived.fim4_featureid_crosswalk\")\n", + " # Drop it already exists\n", + " sql = \"DROP INDEX IF EXISTS derived.fim4_crosswalk_feature_id\"\n", + " cur.execute(sql)\n", + " conn.commit() \n", + " sql = \"CREATE INDEX fim4_crosswalk_feature_id ON derived.fim4_featureid_crosswalk USING btree (feature_id)\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + " print(\"Adding hydro id index to derived.fim4_featureid_crosswalk\")\n", + " # Drop it already exists\n", + " sql = \"DROP INDEX IF EXISTS derived.fim4_crosswalk_hydro_id\"\n", + " cur.execute(sql)\n", + " conn.commit() \n", + " sql = \"CREATE INDEX fim4_crosswalk_hydro_id ON derived.fim4_featureid_crosswalk USING btree (hydro_id)\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + "print(\"\")\n", + "print(f\"Successully loaded derived.fim4_featureid_crosswalk and updated it\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "23500c40-b088-4b62-aa10-fe2f88c52ef9", + "metadata": { + "tags": [] + }, + "source": [ + "

2 - UPDATE FIM HAND PROCESSING LAMBDA ENV VARIABLE WITH NEW FIM PREFIX

\n", + "\n", + "https://us-east-1.console.aws.amazon.com/lambda/home?region=us-east-1#/functions/hv-vpp-ti-viz-hand-fim-processing?tab=configure\n", + "\n", + "Lambda name: hv-vpp-ti-viz-hand-fim-processing\n", + "\n", + "In the Configuration Tab, click on the `Environment variables` (left menu), then change the `FIX_PREFIX` to location of the latest hand_dataset you are working on. Referencial to S3 Bucket name.\n", + "
\n", + "ie) fim/fim_4_5_2_11/hand_datasets\n", + "\n", + "Aug 5, 2024: changed my FIM_PREFIX:\n", + "
from: fim/fim_4_5_2_0/hand_datasets\n", + "
to: fim/fim_4_5_2_11/hand_datasets\n" + ] + }, + { + "cell_type": "markdown", + "id": "040cf1f3-cdfd-467a-9ac0-5478a285f032", + "metadata": { + "tags": [] + }, + "source": [ + "

3 - UPDATE FIM DATA PREP LAMBDA ENV VARIABLE WITH NEW FIM VERSION AND MEMORY

\n", + "\n", + "https://us-east-1.console.aws.amazon.com/lambda/home?region=us-east-1#/functions/hv-vpp-ti-viz-fim-data-prep?tab=code\n", + "\n", + "Lambda name: hv-vpp-ti-viz-hand-fim-processing\n", + "\n", + "In the `Configuration` Tab, click on the `Environment variables` (left menu), then change the `FIM_VERSION` to the latest fim model version. \n", + "
\n", + "ie) 4.5.2.11\n", + "

\n", + "Aug 5, 2024: changed my FIM_VERSION:\n", + "
from: 4.5.2.0\n", + "
to: 4.5.2.11\n", + "

\n", + "Then: Still in the Configuration Tab, now click on the `General Configuration` (left menu), followed \n", + "by the `edit` button on the far right side, to get into the `General Configuration` page details.\n", + "
Change (if they are not already there)\n", + "
Memory (text field) to 4096 (MB) and\n", + "
Emphermeral Storage tp 1024 (MB)\n", + "
\n", + "#### Note: Later in these steps we will change the Memory and Emphermal Storage back to default values, see below ####\n", + "\n", + "Aug 5, 2024: changed my Memory (4096) and Emphermal (1024):\n" + ] + }, + { + "cell_type": "markdown", + "id": "0ef11c15-c1c1-46a8-a65a-fa2a0c1dc97b", + "metadata": { + "tags": [] + }, + "source": [ + "

4 - LOAD AND UPDATE RAS2FIM DATA IN DB

\n", + "\n", + "

Note about ras2fim domain extents

\n", + "As of Aug 2024, a new service came online for a new layer for ras2fim domain extents. Don took care of it.\n", + "The new extent data was loaded as part of different tools and processes, but we will likly want to consolidate\n", + "it to here.\n", + "\n", + "When ras2fim datasets are released, they come with a \"release\" package that has all of the ras2fim models and geocurves\n", + "needed here, but also has domain extents for each HUC in the release package. That entire thing is loaded to S3\n", + "for HV to load. As we will upload a new ras2fim data / geocurves and domain extents at the same time, those load\n", + "scripts should all stay together (here for now). We can add that next time.\n", + "\n", + "However.. ras2fim will likely do releases a lot more regularily than FIM, so it should get it's own independeant load scripts\n", + "which this script can optionally reference if it likes (well.. future versions of this script, ie FIM 4.8.x.x or whatever)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfc0a44c-8b46-4db5-a9a0-c65e0bf1e20b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Already done for 4.4.0.0 (4.5.2.11)\n", + "\n", + "\n", + "# TODO: Aug 2024: Change this to a backup without indexes and not rename\n", + "\n", + "\n", + "# By doing a backup, we are leaving the original tables with the indexes and we want to keep them with\n", + "# ras2fim as it loads geometry and without those pre-existing indexes, loading can fail\n", + "# tbl_name = \"ras2fim.geocurves\"\n", + "# new_table_name = f\"{tbl_name}_{OLD_FIM_TAG}\"\n", + "# sql = f\"CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE {tbl_name};\"\n", + "# sf.execute_sql(sql)\n", + "# print(f\"{tbl_name} copied to {new_table_name} if it does not already exists\")\n", + "\n", + "\n", + "# tbl_name = \"ras2fim.max_geocurves\"\n", + "# new_table_name = f\"{tbl_name}_{OLD_FIM_TAG}\"\n", + "# sql = f\"CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE {tbl_name};\"\n", + "# sf.execute_sql(sql)\n", + "# print(f\"{tbl_name} copied to {new_table_name} if it does not already exists\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e8c3939-f2fc-42de-921d-423d69e023f1", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# NOTE: This can be removed in future ras2fim loads.\n", + "\n", + "\n", + "# Temp Aug 2024: We originally just did a table rename for the {table name} to add _4_4_0_0 on it.\n", + "# Then discovered that renaming it means the indexes are now with the new renamed tables\n", + "# When we load the ras2fim tables, they can't have some of the indexes in place.\n", + "# So.. for now, we are going to rename the _4_4_0_0 tables back to their original name, the do the backup\n", + "# above.\n", + "#sf.execute_sql(f'ALTER TABLE IF EXISTS ras2fim.geocurves_4_4_0_0 RENAME TO geocurves;')\n", + "#sf.execute_sql(f'ALTER TABLE IF EXISTS ras2fim.max_geocurves_4_4_0_0 RENAME TO max_geocurves')\n", + "#print(\"Done renaming them back\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60f910bf-4da7-4e62-b9b6-e981f21a23af", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# This function is not efficient, but as ras2fim has a built in geometry columns, it loads it as a string and not a \"geometry\" object.\n", + "# we have to add the records one at a time.\n", + "\n", + "# Aug 2024: Maybe eventually I can make this more generic, but for now it is ras2fim specific\n", + "# see the new one for catfim as it can likely be rolled into one function\n", + "\n", + "# UPDATE: Sep 19, 2024: We had to remove the chunking portion as we discovered that each csv being loaded might have \n", + "# different crs's. You have to know the incoming crs in order to reproject as the incoming csv's can not be used\n", + "# to auto detect the crs. We put in cards for the ras2fim team to have all final csv's come out as a standard\n", + "# projection (perferraly 3857). Going back to chunking will slow down our DB writes and speed it back up again\n", + "\n", + "# Most of the temp comment code is still in place for chunking.\n", + "\n", + "\n", + "def load_ras2fim_files_into_db(csv_file_list, s3_source_parent_prefix, schema_name, db_name):\n", + "\n", + " # TODO: change these to params and make more generic\n", + " # also tell that this is only if you have a geometry column (for now)\n", + "\n", + " print(f\"Loading data to database {schema_name}.{db_name}\")\n", + " print(\"\")\n", + "\n", + " if len(csv_file_list) == 0:\n", + " raise Exception(\"csv file list is empty\")\n", + "\n", + "\n", + " # source_crs = \"epsg:2277\" # (it is coming in as 5070) but we are changing it to 3857 as loading\n", + "\n", + " # The server has limited memory but it is faster to load as many csv's in at a time\n", + " # as resonablty possible. We are going to try it at chunks of 50 (50 csv files) which for ras2fim\n", + " # shouls be appx 2,000 records, but for ras2fim V2, we have 750 (ish) files.\n", + "\n", + " # We can leave this open the entire times as well.\n", + " s3_client = boto3.client(\"s3\")\n", + " default_kwargs = {\"Bucket\": FIM_BUCKET, \"Prefix\": s3_source_parent_prefix}\n", + "\n", + "\n", + " # chunk_size = 25 # number of csv's to load per set\n", + " total_row_count = 0 # all csv row counts combined. You should see this as a record count in the db when done\n", + " r2f_df = None # a re-used concatenating pd dataframe loading up sets of 20 csvs\n", + " # is_new_df = True # After we db load a set, we reset this to start a new set\n", + " is_first_db_set = True # Very first db load\n", + "\n", + " num_recs = len(csv_file_list)\n", + " print(f\"Total number of files to process are {num_recs}\")\n", + "\n", + " # We are going ot keep the db connection open the entire time. \n", + " # It is slow to open/close connections\n", + " # It \"should not\" ?? block any other scripts / services from usign it\n", + " # most Sql servers allow for more than one connection at a time.\n", + " db = database(db_type=\"viz\")\n", + "\n", + " for idx, full_file_url in enumerate(csv_file_list):\n", + "\n", + " print(f\"Dowloading {idx + 1} of {num_recs} files: {full_file_url}\")\n", + "\n", + " is_first_db_set = idx == 0\n", + " # if idx > 4:\n", + " # return # stub test\n", + "\n", + "# if is_new_df is True:\n", + " s3_client = boto3.client(\"s3\")\n", + " default_kwargs = {\"Bucket\": FIM_BUCKET, \"Prefix\": s3_source_parent_prefix}\n", + "\n", + "\n", + " # is_new_df = False\n", + " # else:\n", + " # temp_df = pd.read_csv(full_file_url)\n", + " # total_row_count += len(temp_df)\n", + " # r2f_df = pd.concat([r2f_df, temp_df])\n", + "\n", + " # we want it merge into the db on each xth (chunk size) record or the last record\n", + " # if ((idx + 1) == num_recs) or ((idx+1) % chunk_size == 0):\n", + "\n", + " # download the csv via pandas into a dataframe\n", + " r2f_df = pd.read_csv(full_file_url)\n", + "\n", + " total_row_count += len(r2f_df)\n", + " r2f_df = r2f_df.fillna(0)\n", + "\n", + " cur_csv = r2f_df.loc[0, 'crs']\n", + " # print(f\"Original crs = {cur_csv}\")\n", + "\n", + " # Create a new source_unit_id which traces back to the folder and code to create\n", + " # this specific huc and model in ras2fim\n", + " r2f_df['source_unit_id'] = r2f_df.apply(lambda row: row.unit_name + \"_\" + \n", + " str(row.unit_version), axis=1)\n", + " r2f_df.rename(columns={'source_code': 'feature_id_source_code', 'geometry': 'geom'}, inplace=True)\n", + " r2f_df['geom'] = r2f_df['geom'].apply(wkt.loads)\n", + "\n", + " # print(f\"... Next set of downloads and adjustments complete, now to db load - Last Idx: {idx + 1} \")\n", + "\n", + " r2f_geodf = gpd.GeoDataFrame(data=r2f_df, geometry='geom', crs=cur_csv)\n", + " # print(r2f_geodf)\n", + " # print(\"\")\n", + " r2f_reproj = r2f_geodf.to_crs(\"epsg:3857\")\n", + "\n", + " # If this is the first load, the type must be the value of \"replace\", else \"append\"\n", + " load_type = 'replace' if is_first_db_set is True else 'append'\n", + "\n", + " r2f_reproj.to_postgis(\n", + " name=db_name,\n", + " con=VIZ_DB_ENGINE,\n", + " schema=schema_name,\n", + " if_exists=load_type,\n", + " index=False,\n", + " )\n", + " print(\"... db load complete\")\n", + "\n", + " # Sanity check on crs\n", + " # if is_first_db_set:\n", + " # print(sf.run_sql_in_db(f\"SELECT ST_SRID(geom) FROM {schema_name}.{db_name} LIMIT 1\"))\n", + "\n", + " r2f_df = None\n", + " r2f_geodf = None\n", + " r2f_reproj = None\n", + " # is_new_df = True # reset it for the next set\n", + " s3_client = None # resets it so it is not open so long. It timse out if open too long\n", + " is_first_db_set = False\n", + "\n", + " # break\n", + "\n", + " # else don't write to db tu continue on to the next record\n", + "\n", + " # end for\n", + " print(\"\")\n", + " print(\"--------------------------------------------------------------\")\n", + " print(\"All records now loaded to the database\")\n", + "\n", + " with db.get_db_connection() as conn, conn.cursor() as cur:\n", + " # after all records are loaded to the db.\n", + " print(f\"Adding {COLUMN_NAME_FIM_VERSION} column to {schema_name}.{db_name}\")\n", + " sql = f\"ALTER TABLE {schema_name}.{db_name} ADD COLUMN IF NOT EXISTS {COLUMN_NAME_FIM_VERSION} text DEFAULT '{PUBLIC_FIM_VERSION}';\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + " print(f\"Adding {COLUMN_NAME_MODEL_VERSION} column to {schema_name}.{db_name}\")\n", + " sql = f\"ALTER TABLE {schema_name}.{db_name} ADD COLUMN IF NOT EXISTS {COLUMN_NAME_MODEL_VERSION} text DEFAULT '{model_version}';\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + " print(\"Dropping un-necessary columns from DB ...\")\n", + " drop_col_names = [\"profile_num\", \"model_id\", \"xs_us\", \"xs_ds\", \"unit_name\", \"unit_version\", \"version\", \"crs\"]\n", + " # print(drop_col_names)\n", + " # print(\"\")\n", + "\n", + " sql = f\"ALTER TABLE {schema_name}.{db_name} \"\n", + " for col_name in drop_col_names:\n", + " sql += f\" DROP COLUMN {col_name},\"\n", + "\n", + " # the last char is a comma and we need to change it to be \" CASCASE;\"\n", + " sql = sql[0:-1] + \" CASCADE;\"\n", + " print(sql)\n", + " cur.execute(sql)\n", + " conn.commit()\n", + " print(f\"Total Rows loaded to DB is {total_row_count}\")\n", + " # end of def\n", + "\n", + "print(\"Download and db load ras2fim function loaded\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e57d7bd1-c852-416e-b259-715b023bf743", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Load the ras2fim.geocurves\n", + "\n", + "# Note: For Aug 2024 (ras 2.0.1.0 with appx 11 hucs, this took appx 1 hr 15 mins to run\n", + "\n", + "model_version = \"ras2fim 2.0.1.0\"\n", + "new_s3_version_folder = \"v2_0\"\n", + "s3_source_parent_prefix = f\"ras2fim/{new_s3_version_folder}\"\n", + "\n", + "start_dt = datetime.now()\n", + "print(\"\")\n", + "print(\"Starting loading of ras2fim.geocurves\")\n", + "\n", + "\n", + "# Aug 21 2024, AWS Creds expired and died just after loading rec 2475 of 7948.\n", + "# Commented out truncate, reset csv_list to be recs 2476 and higher and restarted.\n", + "# All over exact time lost, but can esimate it.\n", + "sql = '''\n", + " TRUNCATE TABLE ras2fim.geocurves;\n", + " TRUNCATE TABLE ras2fim.max_geocurves;\n", + "'''\n", + "print(sf.execute_sql(sql))\n", + "print(\"geocurves and max_geocurves tables truncated to start clean\")\n", + "print(\"\")\n", + "\n", + "# Now download the s3 geocurves\n", + "# Overloaded the server as the memory couldn't handle it.\n", + "# r2f_df = s3_sf.download_S3_csv_files_to_df(FIM_BUCKET, s3_source_parent_prefix, True)\n", + "\n", + "# lets just get a list of files, then iterate over them to load each to the db one at a time.\n", + "r2f_file_names = s3_sf.get_s3_subfolder_file_names(FIM_BUCKET, s3_source_parent_prefix, False)\n", + "\n", + "if len(r2f_file_names) == 0:\n", + " raise Exception(\"No file names found\")\n", + "\n", + "\n", + "csv_file_list = list(filter(lambda x: (x.endswith(\".csv\") == True), r2f_file_names))\n", + "if len(csv_file_list) == 0:\n", + " raise Exception(\"No csv file names found\")\n", + "\n", + "# print(csv_file_list)\n", + "\n", + "# Test against just 20 records for a timing test\n", + "# test_list = csv_file_list[:20]\n", + "# print(test_list)\n", + "\n", + "print(\"Loading df into ras2fim geocurve db\")\n", + "\n", + "load_ras2fim_files_into_db(csv_file_list, s3_source_parent_prefix, 'ras2fim', 'geocurves')\n", + "\n", + "# See note above about having to restart at rec num 2475 (our index displays were 1 based and not zero based\n", + "# restart_list = csv_file_list[974:]\n", + "# load_ras2fim_files_into_db(restart_list, s3_source_parent_prefix, 'ras2fim', 'geocurves')\n", + "\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(\".... ras2fim files now loaded to ras2fim.geocurves\")\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ded5372c-d94e-4276-be77-b13a94db55cc", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# ras2fim \"previous\" columns loading\n", + "\n", + "print(\"... Starting ras2fim previous stage adding and max_geocurves creating\")\n", + "start_dt = datetime.now()\n", + "\n", + "sql = \"\"\"\n", + "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS previous_stage_ft double precision;\n", + "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS previous_stage_m double precision;\n", + "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS previous_discharge_cfs double precision;\n", + "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS previous_discharge_cms double precision;\n", + "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS oid INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY;\n", + "\"\"\" \n", + "print(sf.execute_sql(sql))\n", + "\n", + "sql = '''DROP TABLE IF EXISTS ras2fim.temp_ras2fim_lagged;'''\n", + "print(sf.execute_sql(sql))\n", + "\n", + "\n", + "db = database(db_type=\"viz\")\n", + "with db.get_db_connection() as conn, conn.cursor() as cur:\n", + "\n", + " # PS. It is ok that there are some nulls in the four \"previous\" columns\n", + " sql = \"\"\"\n", + " CREATE TABLE ras2fim.temp_ras2fim_lagged as (SELECT\n", + " feature_id,\n", + " stage_ft,\n", + " (lag(stage_m, 1) OVER (PARTITION BY feature_id ORDER by stage_m)) as previous_stage_m,\n", + " (lag(stage_ft, 1) OVER (PARTITION BY feature_id ORDER by stage_ft)) as previous_stage_ft,\n", + " (lag(discharge_cfs, 1) OVER (PARTITION BY feature_id ORDER by discharge_cfs)) as previous_discharge_cfs,\n", + " (lag(discharge_cms, 1) OVER (PARTITION BY feature_id ORDER by discharge_cms)) as previous_discharge_cms\n", + " FROM ras2fim.geocurves)\n", + " \"\"\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + " sql = \"\"\"\n", + " UPDATE ras2fim.geocurves gc\n", + " SET previous_stage_ft = lagged.previous_stage_ft,\n", + " previous_stage_m = lagged.previous_stage_m,\n", + " previous_discharge_cfs = lagged.previous_discharge_cfs,\n", + " previous_discharge_cms = lagged.previous_discharge_cms\n", + " FROM ras2fim.temp_ras2fim_lagged as lagged\n", + " WHERE gc.feature_id = lagged.feature_id\n", + " and gc.stage_ft = lagged.stage_ft;\n", + " \"\"\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + "\n", + "print(\"Removing ras2fim.temp_ras2fim_lagged table\")\n", + "sql = \"DROP TABLE IF EXISTS ras2fim.temp_ras2fim_lagged;\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "\n", + "print(\"Adding indexes if required\")\n", + "\n", + "sql = \"ALTER TABLE IF EXISTS ras2fim.geocurves OWNER to viz_proc_admin_rw_user;\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "sql = \"ALTER TABLE IF EXISTS ras2fim.geocurves OWNER to viz_proc_admin_rw_user;\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "sql = \"DROP INDEX IF EXISTS ras2fim.geocurves_discharge_cms_index;\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "\n", + "sql = \"\"\"\n", + "CREATE INDEX IF NOT EXISTS geocurves_discharge_cms_index ON ras2fim.geocurves USING btree (discharge_cms ASC NULLS LAST)\n", + "\"\"\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "\n", + "sql = \"DROP INDEX IF EXISTS ras2fim.geocurves_feature_id_index;\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "\n", + "sql = \"\"\"\n", + "CREATE INDEX IF NOT EXISTS geocurves_feature_id_index ON ras2fim.geocurves USING btree (feature_id ASC NULLS LAST)\n", + "\"\"\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "\n", + "sql = \"DROP INDEX IF EXISTS ras2fim.geocurves_previous_discharge_cms_index;\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "\n", + "sql = \"\"\"\n", + "CREATE INDEX IF NOT EXISTS geocurves_previous_discharge_cms_index\n", + " ON ras2fim.geocurves USING btree (previous_discharge_cms ASC NULLS LAST)\n", + "\"\"\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "\n", + "# Skip for now.. not sure if it will be needed in the next set\n", + "# -- DROP INDEX IF EXISTS ras2fim.idx_geocurves_geom;\n", + "# CREATE INDEX IF NOT EXISTS idx_geocurves_geom\n", + "#     ON ras2fim.geocurves USING gist\n", + "#     (geom)\n", + "#     TABLESPACE pg_default;\n", + "\n", + "\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(\".... Done - ras2fim previous stage columns added\")\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1bd997c-f813-49dd-94e0-adf062f3b74b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# ras2fim max_geocurve loading\n", + "\n", + "start_dt = datetime.now()\n", + "\n", + "print(\"Start of creating and loading max_geocurves table\")\n", + "\n", + "# Table can't have any indexes as nothing in unique enough\n", + "# We shoudl have an oid column though\n", + "sql = \"DROP TABLE IF EXISTS ras2fim.max_geocurves;\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "sql = \"\"\"\n", + "CREATE TABLE ras2fim.max_geocurves as (\n", + " SELECT\n", + " feature_id,\n", + " max(discharge_cfs) as max_rc_discharge_cfs,\n", + " max(stage_ft) as max_rc_stage_ft,\n", + " max(discharge_cms) as max_rc_discharge_cms,\n", + " max(stage_m) as max_rc_stage_m\n", + " FROM ras2fim.geocurves\n", + " GROUP BY feature_id )\n", + "\"\"\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "sql = \"DROP INDEX IF EXISTS ras2fim.max_geocurves_feature_id_index;\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "sql = \"\"\"\n", + "CREATE INDEX IF NOT EXISTS max_geocurves_feature_id_index ON \n", + " ras2fim.max_geocurves USING btree (feature_id ASC NULLS LAST);\n", + "\"\"\"\n", + "print(sf.execute_sql(sql))\n", + "\n", + "print(\"max_geocurves table created and filled\")\n", + "\n", + "\n", + "print(\"\")\n", + "\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "bec1b8bf-e1dc-4525-9066-afef7273e289", + "metadata": { + "tags": [] + }, + "source": [ + "

5 - Load the Ras2Fim boundaries into egis

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "251905aa-259a-4ef4-9590-dcd31375c624", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Moves data from local shapefile to EGIS\n", + "\n", + "# Importing Modules\n", + "import os\n", + "import sys\n", + "import helper_functions.shared_functions as sf\n", + "import geopandas as gpd\n", + "\n", + "# sys path if needed\n", + "sys.path.append(os.path.join(os.path.abspath(''), '..'))\n", + "\n", + "# Dir location of your data\n", + "DATA_DPATH = r\"/home/ec2-user/SageMaker/Don - Campground/Don - Store\"\n", + "# File location of your shapefile data\n", + "DATASET_DPATH = f\"{DATA_DPATH}/main_huc8.shp\"\n", + "# Check path by printing\n", + "print(DATASET_DPATH)\n", + "\n", + "# Only use when you want to create something new\n", + "gdf = gpd.read_file(DATASET_DPATH, columns='geometry')\n", + "gdf.to_postgis(name=\"boundaries2\", con=sf.get_db_engine(db_type=\"viz\"),\n", + " schema=\"ras2fim\", if_exists=\"replace\")\n", + "\n", + "# add an oid field to your data\n", + "sql = \"\"\"\n", + "ALTER TABLE ras2fim.boundaries ADD COLUMN oid SERIAL PRIMARY KEY;\n", + "\"\"\"\n", + "sf.execute_sql(sql)\n", + "\n", + "# delete a table in egis if you made a mistake or such\n", + "sql = \"\"\"\n", + "DROP TABLE IF EXISTS reference.ras2fim_boundaries;\n", + "\"\"\"\n", + "sf.execute_sql(sql, db_type=\"egis\")\n", + "\n", + "# The function move_data_from_viz_to_egis is called because the data needs to \n", + "# be in the egis before publishing to uat or prd" + ] + }, + { + "cell_type": "markdown", + "id": "316b1321-f570-4f53-aeee-f8798330642f", + "metadata": { + "tags": [] + }, + "source": [ + "

6 - Run AEP FIM Pipelines.

\n", + "Updated Documentation from Tyler Early 2024: This can be done in a couple of diferent ways.\n", + "\n", + "1) One option is to use the pipeline_input code created below by Corey to start the AEP pipelines directly from this notebook.
\n", + " However, those pipeline_input dictionaries may very well be be out of date, pending more recent updates to the pipelines.\n", + "{\n", + " \"configuration\": \"reference\",\n", + " \"products_to_run\": \"static_nwm_aep_inundation_extent_library\",\n", + " \"invoke_step_function\": false\n", + "}\n", + "\n", + "Using this test event will produce the pipeline instructions, printing any errors that come up, and you can simply change the invoke_step_function flag to True when you're ready to actually invoke a pipeline run (which you can monitor/manage in the step function gui). You will need to manually update the static_nwm_aep_inundation_extent_library.yml product config file to only run 1 aep configuration at a time, and work through the configs as the pipelines finish (takes about an hour each). I've also found that the fim_data_prep lambda function needs to be temporarilly increased to ~4,500mb of memory to run these pipelines. It's also worth noting that these are very resource intesive pipelines, as FIM is calculated for every reach in the nation. AWS costs can amount to hundreds or even thousands of dollars by running these pipelines, so use responsibly.\n", + "\n", + "A couple other important notes:\n", + "- These AEP configurations write data directly to the aep_fim schema in the egis RDS database, instead of the viz database.\n", + "- You'll need to dump the aep_fim schema after that is complete for backup / deployment into other environments.\n", + "- This process has not been tested with new NWM 3.0 Recurrence Flows, and a good thorough audit / QC check of output data is warranted, given those changes and the recent updates to the pipelines.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a698067", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Aug 6, 2024: Note: This was created after all intervals were created, so only HW was tested against\n", + "\n", + "def get_aep_pipeline_input(stage_interval):\n", + " pipeline_input = {\n", + " \"configuration\": \"reference\",\n", + " \"job_type\": \"auto\",\n", + " \"data_type\": \"channel\",\n", + " \"keep_raw\": False,\n", + " \"reference_time\": datetime.now().strftime('%Y-%m-%d 00:00:00'),\n", + " \"configuration_data_flow\": {\n", + " \"db_max_flows\": [],\n", + " \"db_ingest_groups\": [],\n", + " \"python_preprocessing\": []\n", + " },\n", + " \"pipeline_products\": [\n", + " {\n", + " \"product\": \"static_nwm_aep_inundation_extent_library\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": f\"rf_{stage_interval}_inundation\",\n", + " \"target_table\": f\"aep_fim.rf_{stage_interval}_inundation\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": f\"rf_{stage_interval}_inundation\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " \"static_nwm_aep_inundation_extent_library_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " }\n", + " ],\n", + " \"sql_rename_dict\": {},\n", + " \"logging_info\": {\n", + " \"Timestamp\": int(datetime.now().timestamp())\n", + " }\n", + " }\n", + "\n", + " return pipeline_input\n", + "\n", + "print(\"function: get_aep_pipeline_input loaded\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f6d6ee69", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "#### 2 Year Flow\n", + "pipeline_input = get_aep_pipeline_input(\"2\")\n", + "\n", + "# notice, slightly different object name\n", + "pipeline_name = f\"sagemaker_aep_2_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "\n", + "STEPFUNCTION_CLIENT.start_execution(\n", + " stateMachineArn = PIPELINE_ARN,\n", + " name = pipeline_name,\n", + " input= json.dumps(pipeline_input)\n", + ")\n", + "\n", + "print(f\"AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4f89d9a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "#### 5 Year Flow\n", + "pipeline_input = get_aep_pipeline_input(\"5\")\n", + "\n", + "# notice, slightly different object name\n", + "pipeline_name = f\"sagemaker_aep_5_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "\n", + "STEPFUNCTION_CLIENT.start_execution(\n", + " stateMachineArn = PIPELINE_ARN,\n", + " name = pipeline_name,\n", + " input= json.dumps(pipeline_input)\n", + ")\n", + "\n", + "print(f\"AEP : 5 year flows ie: rf_5_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "791d1a8b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "#### 10 Year Flow\n", + "pipeline_input = get_aep_pipeline_input(\"10\")\n", + "\n", + "# notice, slightly different object name\n", + "pipeline_name = f\"sagemaker_aep_10_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "\n", + "STEPFUNCTION_CLIENT.start_execution(\n", + " stateMachineArn = PIPELINE_ARN,\n", + " name = pipeline_name,\n", + " input= json.dumps(pipeline_input)\n", + ")\n", + "\n", + "print(f\"AEP : 10 year flows ie: rf_10_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1bb87128", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "#### 25 Year Flow\n", + "pipeline_input = get_aep_pipeline_input(\"25\")\n", + "\n", + "# notice, slightly different object name\n", + "pipeline_name = f\"sagemaker_aep_25_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "\n", + "STEPFUNCTION_CLIENT.start_execution(\n", + " stateMachineArn = PIPELINE_ARN,\n", + " name = pipeline_name,\n", + " input= json.dumps(pipeline_input)\n", + ")\n", + "\n", + "print(f\"AEP : 25 year flows ie: rf_25_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4832e4e0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "#### 50 Year Flow\n", + "pipeline_input = get_aep_pipeline_input(\"50\")\n", + "\n", + "# notice, slightly different object name\n", + "pipeline_name = f\"sagemaker_aep_50_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "\n", + "STEPFUNCTION_CLIENT.start_execution(\n", + " stateMachineArn = PIPELINE_ARN,\n", + " name = pipeline_name,\n", + " input= json.dumps(pipeline_input)\n", + ")\n", + "\n", + "print(f\"AEP : 50 year flows ie: rf_50_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "187e83bc-ebbe-4615-a046-e0ef7b09ad3b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "#### HW (High Water) Flow\n", + "pipeline_input = get_aep_pipeline_input(\"high_water\")\n", + "\n", + "# notice, slightly different object name\n", + "pipeline_name = f\"sagemaker_aep_hw_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "\n", + "STEPFUNCTION_CLIENT.start_execution(\n", + " stateMachineArn = PIPELINE_ARN,\n", + " name = pipeline_name,\n", + " input= json.dumps(pipeline_input)\n", + ")\n", + "\n", + "print(f\"AEP : High Water year flows ie: rf_hw_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n", + "print(\"\")" + ] + }, + { + "cell_type": "markdown", + "id": "2c810767-2f5d-46b2-860c-5d5c549f2e2a", + "metadata": { + "tags": [] + }, + "source": [ + "

IMPORTANT: Return hv-vpp-ti-viz-fim-data-prep Lambda memory to 2048mb

\n", + "\n", + "https://us-east-1.console.aws.amazon.com/lambda/home?region=us-east-1#/functions/hv-vpp-ti-viz-fim-data-prep?tab=code\n", + "\n", + "Lambda name: hv-vpp-ti-viz-hand-fim-processing\n" + ] + }, + { + "cell_type": "markdown", + "id": "48074dbe-ead7-467f-ae68-16c746115efc", + "metadata": {}, + "source": [ + "

7 - RUN CATCHMENT WORKFLOWS 2 CONFIGS AT A TIME. CHECK FOR STEP FUNCTION FINISHING BEFORE STARTING NEW ONE

" + ] + }, + { + "cell_type": "markdown", + "id": "9b33a346-ee44-4844-9687-627ccd601263", + "metadata": { + "tags": [] + }, + "source": [ + "### 7a - Branch 0 Catchments. Wait until it is done before kicking off the next GMS (Level Path) catchments load a bit lower. ###" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c77d6e5b-f5fb-422f-a6c3-0f53743cd631", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: Add backups to these (4.4.0.0) (already not available for 4.4.0.0)\n", + "\n", + "\n", + "# TODO: We likely need to keep the schema, so trun is fine for now, but eventually, get a lsit of the indexes and re-build \n", + "# indexes each time as/if needed. Granted these tables are loaded via Lambdas, so I am not sure how indexes will play into that\n", + "\n", + "sf.execute_sql('''\n", + "TRUNCATE \n", + " fim_catchments.branch_0_catchments, \n", + " fim_catchments.branch_0_catchments_hi, \n", + " fim_catchments.branch_0_catchments_prvi;\n", + "''', db_type=\"egis\")\n", + "\n", + "print(\"Catchment Truncation for Branch 0 Done\")\n", + "print(\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7495759", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pipeline_input = {\n", + " \"configuration\": \"reference\",\n", + " \"job_type\": \"auto\",\n", + " \"data_type\": \"channel\",\n", + " \"keep_raw\": False,\n", + " \"reference_time\": datetime.now().strftime('%Y-%m-%d 00:00:00'),\n", + " \"configuration_data_flow\": {\n", + " \"db_max_flows\": [],\n", + " \"db_ingest_groups\": [],\n", + " \"python_preprocessing\": []\n", + " },\n", + " \"pipeline_products\": [\n", + " {\n", + " \"product\": \"static_hand_catchments_0_branches\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": \"catchments_0_branches\",\n", + " \"target_table\": \"fim_catchments.branch_0_catchments\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": \"catchments_0_branches\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " \"static_hand_catchments_0_branches_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " },\n", + " {\n", + " \"product\": \"static_hand_catchments_0_branches_hi\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": \"catchments_0_branches_hi\",\n", + " \"target_table\": \"fim_catchments.branch_0_catchments_hi\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": \"catchments_0_branches_hi\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " \"static_hand_catchments_0_branches_hi_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " },\n", + " {\n", + " \"product\": \"static_hand_catchments_0_branches_prvi\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": \"catchments_0_branches_prvi\",\n", + " \"target_table\": \"fim_catchments.branch_0_catchments_prvi\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": \"catchments_0_branches_prvi\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " \"static_hand_catchments_0_branches_prvi_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " }\n", + " ],\n", + " \"sql_rename_dict\": {},\n", + " \"logging_info\": {\n", + " \"Timestamp\": int(datetime.now().timestamp())\n", + " }\n", + "}\n", + "\n", + "pipeline_name = f\"sagemaker_0_catchments_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "\n", + "# TODO: For later... fix fim_version value and add model_version column. current fim_version vlaue is showing 4.5.2.11\n", + "\n", + "STEPFUNCTION_CLIENT.start_execution(\n", + " stateMachineArn = PIPELINE_ARN,\n", + " name = pipeline_name,\n", + " input= json.dumps(pipeline_input)\n", + ")\n", + "\n", + "print(\"Catchments Branch 0 load kicked off. Last runtime: 23:38.019. \"\n", + " f\"Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "d792a88e-51fd-4e28-9360-b90abd99ddaf", + "metadata": {}, + "source": [ + "### 7b - GMS (Level Paths / non branch 0) catchments ###" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ea2a5be-9936-4abe-832a-8582f3c3af64", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# TODO: Add backups to these\n", + "\n", + "sf.execute_sql('''\n", + "TRUNCATE\n", + " fim_catchments.branch_gms_catchments,\n", + " fim_catchments.branch_gms_catchments_hi,\n", + " fim_catchments.branch_gms_catchments_prvi;\n", + "''', db_type=\"egis\")\n", + "\n", + "print(\"Catchment Truncation for GMS (Level Path) Branchs Done\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ae8917f", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pipeline_input = {\n", + " \"configuration\": \"reference\",\n", + " \"job_type\": \"auto\",\n", + " \"data_type\": \"channel\",\n", + " \"keep_raw\": False,\n", + " \"reference_time\": datetime.now().strftime('%Y-%m-%d 00:00:00'),\n", + " \"configuration_data_flow\": {\n", + " \"db_max_flows\": [],\n", + " \"db_ingest_groups\": [],\n", + " \"python_preprocessing\": []\n", + " },\n", + " \"pipeline_products\": [\n", + " {\n", + " \"product\": \"static_hand_catchments_gms_branches\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": \"catchments_gms_branches\",\n", + " \"target_table\": \"fim_catchments.branch_gms_catchments\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": \"catchments_gms_branches\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " \"static_hand_catchments_gms_branches_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " },\n", + " {\n", + " \"product\": \"static_hand_catchments_gms_branches_hi\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": \"catchments_gms_branches_hi\",\n", + " \"target_table\": \"fim_catchments.branch_gms_catchments_hi\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": \"catchments_gms_branches_hi\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " \"static_hand_catchments_gms_branches_hi_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " },\n", + " {\n", + " \"product\": \"static_hand_catchments_gms_branches_prvi\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": \"catchments_gms_branches_prvi\",\n", + " \"target_table\": \"fim_catchments.branch_gms_catchments_prvi\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": \"catchments_gms_branches_prvi\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " \"static_hand_catchments_gms_branches_prvi_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " }\n", + " ],\n", + " \"sql_rename_dict\": {},\n", + " \"logging_info\": {\n", + " \"Timestamp\": int(datetime.now().timestamp())\n", + " }\n", + "}\n", + "\n", + "pipeline_name = f\"sagemaker_gms_catchments_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "\n", + "# TODO: For later... fix fim_version value and add model_version column. current fim_version vlaue is showing 4.5.2.11\n", + "\n", + "STEPFUNCTION_CLIENT.start_execution(\n", + " stateMachineArn = PIPELINE_ARN,\n", + " name = pipeline_name,\n", + " input= json.dumps(pipeline_input)\n", + ")\n", + "\n", + "print(\"Catchments GMS Branches (Level Paths / non branch 0) load kicked off.\"\n", + " f\" Last runtime: 24:45.150. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "6ff4dc6b-c41b-412c-9d9f-d812c5c72115", + "metadata": { + "tags": [] + }, + "source": [ + "

8 - Recreate derived.usgs_elev_table

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e4eb95e-e147-4789-a76b-d8bda33bc3cb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Already run for 4.4.0.0 (4.5.2.11)\n", + "\n", + "# TODO: Aug 2024: Change this to a backup without indexes and not rename\n", + "\n", + "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.usgs_elev_table RENAME TO usgs_elev_table_fim_{OLD_FIM_TAG};')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# NOTE: Aug 6, 2024 - There is already a 4.4.0.0 backup. How did it get made?\n", + "# Has appx 2,150 HUCs to process, but this section goes quickly.\n", + "\n", + "sf.execute_sql('DROP TABLE IF EXISTS derived.usgs_elev_table;')\n", + "\n", + "uet_usecols = ['location_id', 'HydroID', 'dem_adj_elevation', 'nws_lid', 'levpa_id']\n", + "\n", + "paginator = S3_CLIENT.get_paginator('list_objects')\n", + "operation_parameters = {'Bucket': FIM_BUCKET,\n", + " 'Prefix': f'{HAND_DATASETS_DPATH}/',\n", + " 'Delimiter': '/'}\n", + "page_iterator = paginator.paginate(**operation_parameters)\n", + "page_count = 0\n", + "for page in page_iterator:\n", + " page_count += 1\n", + " prefix_objects = page['CommonPrefixes']\n", + " for i, prefix_obj in enumerate(prefix_objects):\n", + " print(f\"Processing {i+1} of {len(prefix_objects)} on page {page_count}\")\n", + " huc_prefix = prefix_obj.get(\"Prefix\")\n", + " usgs_elev_table_key = f'{huc_prefix}usgs_elev_table.csv'\n", + " try:\n", + " uet = S3_CLIENT.get_object(\n", + " Bucket=FIM_BUCKET, \n", + " Key=usgs_elev_table_key\n", + " )['Body']\n", + " uet_df = pd.read_csv(uet, header=0, usecols=uet_usecols)\n", + " uet_df['fim_version'] = PUBLIC_FIM_VERSION\n", + " uet_df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", + " uet_df.to_sql(\n", + " con=VIZ_DB_ENGINE,\n", + " dtype={\n", + " \"location_id\": Text(),\n", + " \"nws_data_huc\": Text()\n", + " },\n", + " schema='derived',\n", + " name='usgs_elev_table',\n", + " index=False, \n", + " if_exists='append'\n", + " )\n", + " except Exception as e:\n", + " if \"NoSuchKey\" in str(e):\n", + " pass\n", + " else:\n", + " raise e\n", + " \n", + "print(\"usgs_elev_tables load completed\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "92a02c35-16a9-4cd8-9fd6-61d20b3e6feb", + "metadata": {}, + "source": [ + "

9 - Recreate derived.hydrotable_staggered

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18a2a7c2-7499-4ebc-a878-769e62dfbd6b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Already run for 4.4.0.0 (4.5.2.11)\n", + "\n", + "# TODO: Aug 2024: Change this to a backup without indexes and not rename\n", + "\n", + "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.hydrotable_staggered RENAME TO hydrotable_staggered_{OLD_FIM_TAG};')\n", + "#print(\"derived.hydrotable_staggered renamed\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33026eb6-fbc7-47f7-a44b-f9cea8f0911b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Takes appx 5.75 to 6 hrs to run\n", + "\n", + "print(\"hydrotable reloaded - started\")\n", + "start_dt = datetime.now()\n", + "\n", + "sf.execute_sql('DROP TABLE IF EXISTS derived.hydrotable;')\n", + "sql = '''\n", + "SELECT distinct LPAD(huc8::text, 8, '0') as huc8 FROM derived.featureid_huc_crosswalk WHERE huc8 is not null;\n", + "'''\n", + "df = sf.sql_to_dataframe(sql)\n", + "ht_usecols = ['HydroID', 'feature_id', 'stage', 'discharge_cms']\n", + "\n", + "paginator = S3_CLIENT.get_paginator('list_objects')\n", + "operation_parameters = {'Bucket': FIM_BUCKET,\n", + " 'Prefix': f'{HAND_DATASETS_DPATH}/',\n", + " 'Delimiter': '/'}\n", + "page_iterator = paginator.paginate(**operation_parameters)\n", + "page_count = 0\n", + "for page in page_iterator:\n", + " page_count += 1\n", + " prefix_objects = page['CommonPrefixes']\n", + " for i, prefix_obj in enumerate(prefix_objects):\n", + " print(f\"Processing {i+1} of {len(prefix_objects)} on page {page_count}\")\n", + " branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", + " branch_files_result = S3_CLIENT.list_objects(\n", + " Bucket=FIM_BUCKET, \n", + " Prefix=branch_prefix, \n", + " Delimiter='/'\n", + " )\n", + " hydro_table_key = None\n", + " for content_obj in branch_files_result.get('Contents'):\n", + " branch_file_prefix = content_obj['Key']\n", + " if 'hydroTable' in branch_file_prefix:\n", + " hydro_table_key = branch_file_prefix\n", + "\n", + " if hydro_table_key:\n", + " # print(f\"Found usgs_elev_table and hydroTable in {branch_prefix}\")\n", + " try:\n", + " # print(\"...Fetching csvs...\")\n", + " ht = S3_CLIENT.get_object(\n", + " Bucket=FIM_BUCKET,\n", + " Key=hydro_table_key\n", + " )['Body']\n", + " # print(\"...Reading with pandas...\")\n", + " ht_df = pd.read_csv(ht, header=0, usecols=ht_usecols)\n", + " # print('...Writing to db...')\n", + " ht_df['fim_version'] = PUBLIC_FIM_VERSION\n", + " ht_df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", + " ht_df.to_sql(\n", + " con=VIZ_DB_ENGINE, \n", + " schema='derived',\n", + " name='hydrotable',\n", + " index=False,\n", + " if_exists='append'\n", + " )\n", + " except Exception as e:\n", + " raise e\n", + " print(f'Fetch failed: {e}')\n", + " \n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(\"hydrotable reload done\")\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5cc764e4-7e67-4b51-a100-6021e31416cd", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "print(\"hydrotable_staggered started\")\n", + "\n", + "start_dt = datetime.now()\n", + "\n", + "sql = '''\n", + "DROP TABLE IF EXISTS derived.hydrotable_staggered;\n", + "SELECT\n", + " et.location_id,\n", + " ht.feature_id,\n", + " (stage + et.dem_adj_elevation) * 3.28084 as elevation_ft,\n", + " LEAD((stage + et.dem_adj_elevation) * 3.28084) OVER (PARTITION BY ht.feature_id ORDER BY ht.feature_id, stage) as next_elevation_ft,\n", + " discharge_cms * 35.3147 as discharge_cfs,\n", + " LEAD(discharge_cms * 35.3147) OVER (PARTITION BY ht.feature_id ORDER BY ht.feature_id, stage) as next_discharge_cfs\n", + "INTO derived.hydrotable_staggered\n", + "FROM derived.hydrotable AS ht\n", + "JOIN derived.usgs_elev_table AS et ON ht.\"HydroID\" = et.\"HydroID\" AND et.location_id IS NOT NULL;\n", + "'''\n", + "sf.execute_sql(sql)\n", + "\n", + "print(\"hydrotable_staggered reload done\")\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74f21c03-ca36-402c-a69b-396162720f66", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# we don't need the hydrotable anymore as it has been reloaded and adjusted above in hydrotable_staggered\n", + "sf.execute_sql('DROP TABLE IF EXISTS derived.hydrotable;')\n", + "print(\"Done dropping derived.hydrotable, post hydrotable_staggered load\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "0237f374-5829-4050-b503-958c9d4c3703", + "metadata": { + "tags": [] + }, + "source": [ + "

10 - Recreate derived.usgs_rating_curves_staggered

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "461f20b4-8442-4c0e-9114-ea62764cd789", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Aug 16, 2024 - done for 4.4.0.0 (4.5.2.11)\n", + "\n", + "# TODO: Aug 2024: Change this to a backup without indexes and not rename\n", + "# Aug 27, 2024: This needs to be redone so we don't rename tables, it messes up indexes and index names when we use _to_sql commands later\n", + "\n", + "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.usgs_rating_curves RENAME TO usgs_rating_curves_{OLD_FIM_TAG};')\n", + "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.usgs_rating_curves_staggered RENAME TO usgs_rating_curves_staggered_{OLD_FIM_TAG};')\n", + "# print(\"usgs rating curve tables renamed and cleaned\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b752fce5-c8d7-4064-92d7-842c22d1723e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "sql = '''\n", + " DROP TABLE IF EXISTS derived.usgs_rating_curves;\n", + " DROP TABLE IF EXISTS derived.usgs_rating_curves_staggered;\n", + "'''\n", + "sf.execute_sql(sql)\n", + "\n", + "print(\"Done dropping usgs_rating_curves and usgs_rating_curves_staggered\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6a0cc18-de82-4bd8-bd4d-c3ad021a1dec", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# run the script to load the usgs_rating_curve.csv. Exact duration not yet known. Appx 30 min (??)\n", + "\n", + "start_dt = datetime.now()\n", + "event = {\n", + " 'target_table': 'derived.usgs_rating_curves',\n", + " 'target_cols': ['location_id', 'flow', 'stage', 'navd88_datum', 'elevation_navd88'],\n", + " 'file': f'{QA_DATASETS_DPATH}/usgs_rating_curves.csv',\n", + " 'bucket': FIM_BUCKET,\n", + " 'reference_time': '2023-08-23 00:00:00',\n", + " 'keep_flows_at_or_above': 0,\n", + " 'iteration_index': 0\n", + "}\n", + "\n", + "sf.execute_db_ingest(event, None)\n", + "\n", + "print(\"done loading usgs_rating_curves\")\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4548d5e8-303e-44e0-a8a5-7b1947214b4e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Takes under a minute\n", + "print(\"Starting usgs_rating_curves_staggered build based on usgs_rating_curve table\")\n", + "\n", + "sql = '''\n", + "SELECT \n", + " location_id,\n", + " flow as discharge_cfs, \n", + " LEAD(flow) OVER (PARTITION BY location_id ORDER BY location_id, stage) as next_discharge_cfs,\n", + " stage,\n", + " navd88_datum,\n", + " elevation_navd88 as elevation_ft,\n", + " LEAD(elevation_navd88) OVER (PARTITION BY location_id ORDER BY location_id, stage) as next_elevation_ft\n", + "INTO derived.usgs_rating_curves_staggered\n", + "FROM derived.usgs_rating_curves;\n", + "'''\n", + "\n", + "sf.execute_sql(sql)\n", + "\n", + "print(\"Done loading usgs_rating_curves_staggered\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1a77875-f569-4410-888c-b132714a7c9e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# usgs_rating_curves is a temp table and is loaded with some changes into the usgs_rating_curves_staggered\n", + "sf.execute_sql('DROP TABLE IF EXISTS derived.usgs_rating_curves;')\n", + "print(\"Done dropping derived.usgs_rating_curves, post loading usgs_rating_curves_staggered\")" + ] + }, + { + "cell_type": "markdown", + "id": "ee640076-be6f-47f5-b091-f60192502bd0", + "metadata": { + "tags": [] + }, + "source": [ + "

11 - UPDATE SRC SKILL METRICS IN DB

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95c472f8-0557-4864-8536-1814f3ac4286", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Already run for 4.4.0.0 (4.5.2.11)\n", + "\n", + "'''\n", + "Be Very Careful to just rename tables. If they have indexes, the index will now point to the new\n", + "table names but maintain the original index name. Those index names can really mess stuff up.\n", + "Best to never rename unless you rename indexes as well. This particular on is ok. \n", + "Note: When various '\"to_sql\" tools are run which have GIST indexes, this index column name issue\n", + "will be the problem.\n", + "\n", + "Why Drop instead of Truncate? if the schema changes for the incoming, truncate will have column\n", + "missmatches.\n", + "\n", + "We really should be backing up indexes and constraints as well.\n", + "\n", + "'''\n", + "\n", + "# TODO: Aug 2024: Change this away from \"rename\" to copy / drop. \n", + "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.src_skill_temp RENAME TO src_skill_temp_{OLD_FIM_TAG};')\n", + "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.src_skill RENAME TO src_skill_{OLD_FIM_TAG};')\n", + "\n", + "# print(\"src_skill and src_skill_temps db renamed\")\n", + "\n", + "\n", + "# TODO: Rob Aug 2024: change this to backup of table and not rename as it messses with indexes\n", + "# Don't need a copy of the reference src_skill table , so just drop it.\n", + "new_table_name = f\"derived.src_skill_temp_{OLD_FIM_TAG}\"\n", + "sql = f'''\n", + " CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE derived.src_skill_temp;\n", + "'''\n", + "\n", + "\n", + "#print(\"src_skill and src_skill_temps db renamed\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a249854-b9cd-4ccb-afea-96c35708515d", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Prep the dbs for the new load\n", + "#sf.execute_sql('DROP TABLE IF EXISTS derived.src_skill_temp;')\n", + "#sf.execute_sql('DROP TABLE IF EXISTS reference.src_skill;', db_type='egis')\n", + "#print(\"Done dropping src_skill and src_skill_temp tables\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c24f509a-90df-492a-a0d4-aa9eab8e7041", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Load the src_skill_temp table\n", + "start_dt = datetime.now()\n", + "\n", + "event = {\n", + " 'target_table': 'derived.src_skill_temp',\n", + " 'target_cols': None, # This means \"all\"\n", + " 'file': f'{QA_DATASETS_DPATH}/agg_nwm_recurr_flow_elev_stats_location_id.csv',\n", + " 'bucket': FIM_BUCKET,\n", + " 'reference_time': '2023-08-23 00:00:00',\n", + " 'keep_flows_at_or_above': 0,\n", + " 'iteration_index': 0,\n", + " 'db_type': 'viz'\n", + "}\n", + "\n", + "execute_db_ingest(event, None)\n", + "print(\"Done loading derived.src_skill_temp table\")\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5639447-9a5a-4add-9c36-ebd945a5a8d0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Load into src_skill table adding geometry to it from external.usgs_gage. Yes.. more/less straight from WRDS tables\n", + "# Some recs appear to be in error in the csv. location id = 394220106431500 (those are dropped below)\n", + "\n", + "start_dt = datetime.now()\n", + "\n", + "sf.execute_sql('DROP TABLE IF EXISTS derived.src_skill;')\n", + "\n", + "sql = f\"\"\"\n", + "SELECT\n", + "\t(row_number() OVER ())::int as oid,\n", + "\tgage.name,\n", + "\tLPAD(skill.location_id::text, 8, '0') as location_id,\n", + "\tskill.nrmse,\n", + "\tskill.mean_abs_y_diff_ft,\n", + "\tskill.mean_y_diff_ft,\n", + "\tskill.percent_bias,\n", + " '{PUBLIC_FIM_VERSION}' as {COLUMN_NAME_FIM_VERSION},\n", + " '{FIM_MODEL_VERSION}' as {COLUMN_NAME_MODEL_VERSION},\n", + "\tgage.geo_point as geom\n", + "INTO derived.src_skill\n", + "FROM derived.src_skill_temp skill\n", + "JOIN external.usgs_gage AS gage ON LPAD(gage.usgs_gage_id::text, 8, '0') = LPAD(skill.location_id::text, 8, '0')\n", + "\"\"\"\n", + "\n", + "sf.execute_sql(sql)\n", + "\n", + "print(\"Done loading derived.src_skill table\")\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "ffa32cd0-6edc-4441-9be3-f5bcba499ad0", + "metadata": { + "tags": [] + }, + "source": [ + "

Then export the derived.src_skill table and import it into the EGIS reference.src_skill table

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0df23345-47a4-490a-b10f-2020e3a2a68e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "sf.move_data_from_viz_to_egis(\"derived.src_skill\", \"reference.src_skill\")\n", + "print(\"Done\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "cf47b262-d5aa-41e3-a12e-99432f52fbfc", + "metadata": { + "tags": [] + }, + "source": [ + "

12 - UPDATE FIM PERFORMANCE METRICS IN DB

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b291a461-e36f-4049-84b6-dd59cd1c4ad2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Make copies of current dbs for 4.4.0.0 (4.5.2.11)\n", + "# DONE: for 4.4.0.0 (4.5.2.11)\n", + "\n", + "# NOTE: Aug 2024: The problem with not droppign them and rebuilding them with indexes, is that if the table schema\n", + "# changes it is not reflected\n", + "\n", + "\n", + "# Points\n", + "new_table_name = f\"reference.fim_performance_points_{OLD_FIM_TAG}\"\n", + "sql = f'''\n", + " CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.fim_performance_points;\n", + "'''\n", + "sf.execute_sql(sql, db_type='egis')\n", + "print(f\"fim_performance_points copied to {new_table_name} if it does not already exists\")\n", + "\n", + "\n", + "# Catchments\n", + "new_table_name = f\"reference.fim_performance_catchments_{OLD_FIM_TAG}\"\n", + "sql = f'''\n", + " CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.fim_performance_catchments;\n", + "'''\n", + "sf.execute_sql(sql, db_type='egis')\n", + "print(f\"fim_performance_catchments copied to {new_table_name} if it does not already exists\")\n", + "\n", + "\n", + "# Polys\n", + "new_table_name = f\"reference.fim_performance_polys_{OLD_FIM_TAG}\"\n", + "sql = f'''\n", + " CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.fim_performance_polys;\n", + "'''\n", + "sf.execute_sql(sql, db_type='egis')\n", + "print(f\"fim_performance_polys copied to {new_table_name} if it does not already exists\")\n", + "\n", + "print(\"Done making backups of the FIM performance tables\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4fcdfab-3936-4a0a-8ab9-605e9a64f9bd", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# clean up tables for new load\n", + "\n", + "# TODO: Aug 2024: Add postgresql if / else. Truncate \"if exists\" doesn't exist. :)\n", + "\n", + "table_names = [\n", + " \"reference.fim_performance_points\",\n", + " \"reference.fim_performance_polys\",\n", + " \"reference.fim_performance_catchments\"\n", + "]\n", + "\n", + "for tb_name in table_names:\n", + " sql = f\"TRUNCATE TABLE {tb_name}\"\n", + "# print(sql)\n", + " sf.execute_sql(sql,db_type='egis')\n", + "\n", + "\n", + "print(f\"All fim_performance tables trunated if they exist\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06ca5581", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "# Load the new fim performance tables\n", + "\n", + "start_dt = datetime.now()\n", + "\n", + "# os.environ['EGIS_DB_HOST'] ='' #TI DB\n", + "\n", + "db_type = \"egis\"\n", + "db_engine = sf.get_db_engine(db_type)\n", + "s3 = boto3.client('s3')\n", + "\n", + "# Define bucket and parent directories.\n", + "bucket = \"hydrovis-ti-deployment-us-east-1\"\n", + "\n", + "# file_handles = ['fim_performance_points.csv']\n", + "# file_handles = ['fim_performance_points.csv', 'fim_performance_polys.csv', 'fim_performance_catchments_dissolved.csv']\n", + "# file_handles = ['fim_performance_points.csv', 'fim_performance_polys.csv']\n", + "file_handles = ['fim_performance_catchments.csv']\n", + "\n", + "for file_handle in file_handles:\n", + "\n", + " print(\"Reading file...\")\n", + " # df = pd.read_csv(local_download_path)\n", + " file_to_download = f\"{QA_DATASETS_DPATH}/{file_handle}\"\n", + " df = s3_sf.download_S3_csv_files_to_df_from_list(FIM_BUCKET, [file_to_download], True)\n", + " print(\"File read.\")\n", + "\n", + " # Rename headers.\n", + "\n", + " if file_handle == 'fim_performance_points.csv':\n", + " df = df.rename(columns={'Unnamed: 0': 'oid', 'geometry': 'geom'})\n", + " else:\n", + " df = df.rename(columns={'Unnamed: 0': 'oid', 'geometry': 'geom', 'huc':'huc8'})\n", + "\n", + " print(df.dtypes)\n", + " # Convert all field names to lowercase (needed for ArcGIS Pro).\n", + " df.columns = df.columns.str.lower()\n", + "\n", + " # Enforce data types on df before loading in DB (TODO: need to create special cases for each layer).\n", + " if file_handle == 'fim_performance_points.csv':\n", + " df = df.astype({'huc': 'str'})\n", + " else:\n", + " df = df.astype({'huc8': 'str'})\n", + " df = df.fillna(0)\n", + " try:\n", + " df = df.astype({'feature_id': 'int'})\n", + " df = df.astype({'feature_id': 'str'})\n", + " df = df.astype({'oid': 'int'})\n", + " except KeyError: # If there is no feature_id field\n", + " pass\n", + " try:\n", + " df = df.astype({'nwm_seg': 'int'})\n", + " df = df.astype({'nwm_seg': 'str'})\n", + " except KeyError: # If there is no nwm_seg field\n", + " pass\n", + " try:\n", + " df = df.astype({'usgs_gage': 'int'})\n", + " df = df.astype({'usgs_gage': 'str'})\n", + " except KeyError: # If there is no usgs_gage field\n", + " pass\n", + "\n", + " # zfill HUC8 field.\n", + " if file_handle == 'fim_performance_points.csv':\n", + " df['huc'] = df['huc'].apply(lambda x: x.zfill(8))\n", + " else:\n", + " df['huc8'] = df['huc8'].apply(lambda x: x.zfill(8))\n", + "\n", + " df['version'] = PUBLIC_FIM_VERSION\n", + " df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", + "\n", + " # Upload df to database.\n", + " stripped_layer_name = file_handle.replace(\".csv\", \"\")\n", + " table_name = \"reference.\" + stripped_layer_name\n", + " print(\"Loading data into DB...\")\n", + "\n", + " # Chunk load data into DB\n", + "\n", + " if file_handle in ['fim_performance_catchments.csv']:\n", + "\n", + " print(\"Chunk loading...\")\n", + " # Create list of df chunks\n", + " n = 10000 # chunk row size\n", + " list_df = [df[i:i+n] for i in range(0,df.shape[0],n)]\n", + " # geometry = 'MULTIPOLYGON'\n", + " # Load the first chunk into the DB as a new table\n", + " first_chunk_df = list_df[0]\n", + " print(first_chunk_df.shape[0])\n", + "\n", + " first_chunk_df.to_sql(\n", + " name=stripped_layer_name, \n", + " con=db_engine, \n", + " schema='reference',\n", + " if_exists='replace', \n", + " index=False,\n", + " dtype={'oid': sqlalchemy.types.Integer(),\n", + " 'version': sqlalchemy.types.String(),\n", + " 'geom': Geometry('MULTIPOLYGON', srid=3857)\n", + " }\n", + " )\n", + " # Load remaining chunks into newly created table\n", + "\n", + " for remaining_chunk_df in list_df[1:]:\n", + " print(remaining_chunk_df.shape[0])\n", + " remaining_chunk_df.to_sql(\n", + " name=stripped_layer_name,\n", + " con=db_engine,\n", + " schema='reference',\n", + " if_exists='append',\n", + " index=False,\n", + " dtype={'oid': sqlalchemy.types.Integer(),\n", + " 'version': sqlalchemy.types.String(),\n", + " 'geom': Geometry('MULTIPOLYGON', srid=3857)\n", + " }\n", + " )\n", + " else:\n", + " if 'points' in stripped_layer_name: geometry = 'POINT'\n", + " if 'polys' in stripped_layer_name: geometry = 'POLYGON'\n", + " # print(\"GEOMETRY\")\n", + " # print(geometry)\n", + " df.to_sql(\n", + " name=stripped_layer_name,\n", + " con=db_engine,\n", + " schema='reference',\n", + " if_exists='replace',\n", + " index=False,\n", + " dtype={'oid': sqlalchemy.types.Integer(),\n", + " 'version': sqlalchemy.types.String(),\n", + " 'geom': Geometry(geometry, srid=3857)\n", + " }\n", + " )\n", + "\n", + " print(f\">>> {file_handle} downloaded and loaded\")\n", + "\n", + " # deleted the downloaded file that was just processed.\n", + " # if os.path.exists(local_download_path):\n", + "\n", + "\n", + "print(\"\")\n", + "\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "# print(\"All FIM Performance files loaded\")\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "a52c1563-3c91-4810-96d0-48568b60cbef", + "metadata": { + "tags": [] + }, + "source": [ + "

13 - CatFIM (Stage-Based and Flow-Based)

" + ] + }, + { + "cell_type": "markdown", + "id": "7938fcb2-313c-4cf8-8274-d34d80bdf7bf", + "metadata": {}, + "source": [ + "

Function to load CatFIM Data (Non Public)

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37f37c81-d105-4c38-aa49-b4aef40a7543", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "''' Function to load CatFIM data (for any flow / stage / library / sites but non public)'''\n", + "\n", + "\n", + "def load_catfim_table(catfim_type):\n", + "\n", + " '''\n", + " Inputs:\n", + " - catfim_type: name identififer for the set, such as \"flow_based_catfim\" or \"flow_based_catfim_sites\", etc\n", + " Sometimes the file_handle name can be the name of the s3 file (without extension) and/or the table\n", + " name.\n", + " Options: flow_based_catfim, flow_based_catfim_sites, stage_based_catfim, stage_based_catfim_sites\n", + " '''\n", + "\n", + " db_type = \"egis\"\n", + " db_engine = sf.get_db_engine(db_type)\n", + " src_crs = \"3857\"\n", + "\n", + " # --------------------------------------\n", + " # Drop the original Db if already in place\n", + " table_name = catfim_type # yes, dup variable for now\n", + "\n", + " sf.execute_sql(f\"DROP TABLE IF EXISTS reference.{table_name};\", db_type=db_type)\n", + " print(f\"Dropping reference.{table_name} table if it existed\")\n", + " print(\"\")\n", + "\n", + " # --------------------------------------\n", + " # Get the data from S3 and load it into a df\n", + " if catfim_type in ['flow_based_catfim', 'stage_based_catfim']:\n", + " file_to_download = f\"{QA_DATASETS_DPATH}/{catfim_type}_library.csv\"\n", + " else:\n", + " file_to_download = f\"{QA_DATASETS_DPATH}/{catfim_type}.csv\"\n", + "\n", + " # print(f\"Downloading {file_to_download} ... \")\n", + "\n", + " df = s3_sf.download_S3_csv_files_to_df_from_list(FIM_BUCKET, [file_to_download], True)\n", + " num_recs = len(df)\n", + " print(f\"File read. {num_recs} records to load\")\n", + "\n", + " # --------------------------------------\n", + " # Adjusting Columns and data\n", + " # Rename headers. All files this name\n", + " df = df.rename(columns={'Unnamed: 0': 'oid',\n", + " 'geometry': 'geom',\n", + " 'huc': 'huc8'})\n", + "\n", + " # 4.5.2.11, fixing a column name bug\n", + " if catfim_type == 'stage_based_catfim_sites':\n", + " df = df.rename(columns={'nws_lid': 'ahps_lid'})\n", + "\n", + " # Convert all field names to lowercase (needed for ArcGIS Pro).\n", + " df.columns = df.columns.str.lower()\n", + "\n", + " # Remove sites that are in derived.ahps_restricted_sites\n", + " # TODO: Aug 2024: Need to see if this list needs to be updated. Submitted card.\n", + " restricted_sites_df = sf.get_db_values(\"derived.ahps_restricted_sites\", [\"*\"])\n", + " restricted_dict = restricted_sites_df.to_dict('records')\n", + "\n", + " for site in restricted_dict:\n", + " nws_lid = site['nws_lid'].lower()\n", + " #print(nws_lid)\n", + " if \"sites\" in catfim_type:\n", + " # print(True)\n", + " # print(nws_lid)\n", + " df.loc[df.ahps_lid == nws_lid, 'mapped'] = 'no'\n", + " df.loc[df.ahps_lid == nws_lid, 'status'] = site['restricted_reason']\n", + " # print(df.loc[df.ahps_lid==nws_lid]['status'])\n", + " else:\n", + " df.loc[df.ahps_lid == nws_lid, 'viz'] = 'no'\n", + " df = df[df['viz'] == 'yes']\n", + "\n", + " # TODO: Aug 2024: This may be a bug or very outdated. It was in the code to load stage for 4.4.0.0\n", + " # and I left it here for 4.5.2.11, but made a card with the FIM team to review and fix it in there code\n", + " # so we can drop this.\n", + " if 'stage_based' in catfim_type:\n", + " for sea_level_site in ['qutg1', 'augg1', 'baxg1', 'lamf1', 'adlg1', 'hrag1', 'stng1']:\n", + " if \"sites\" in catfim_type:\n", + " df.loc[df.ahps_lid==sea_level_site, 'mapped'] = 'no'\n", + " df.loc[df.ahps_lid==sea_level_site, 'status'] = 'Stage thresholds seem to be based on sea level and not channel thalweg'\n", + " else:\n", + " df.loc[df.ahps_lid==sea_level_site, 'viz'] = 'no'\n", + " df = df[df['viz']=='yes'] # Subset df to only sites desired for mapping\n", + " # end if\n", + "\n", + " # Enforce data types on df before loading in DB (TODO: need to create special cases for each layer).\n", + " df = df.astype({'huc8': 'str'})\n", + " df = df.fillna(0)\n", + " try:\n", + " df = df.astype({'feature_id': 'int'})\n", + " df = df.astype({'feature_id': 'str'})\n", + " except KeyError: # If there is no feature_id field\n", + " pass\n", + " try:\n", + " df = df.astype({'nwm_seg': 'int'})\n", + " df = df.astype({'nwm_seg': 'str'})\n", + " except KeyError: # If there is no nwm_seg field\n", + " pass\n", + " try:\n", + " df = df.astype({'usgs_gage': 'int'})\n", + " df = df.astype({'usgs_gage': 'str'})\n", + " except KeyError: # If there is no usgs_gage field\n", + " pass\n", + "\n", + " # zfill HUC8 field.\n", + " df['huc8'] = df['huc8'].apply(lambda x: x.zfill(8))\n", + "\n", + " if '_sites' in catfim_type:\n", + " df = df.astype({'nws_data_rfc_forecast_point': 'str'})\n", + " df = df.astype({'nws_data_rfc_defined_fcst_point': 'str'})\n", + " df = df.astype({'nws_data_riverpoint': 'str'})\n", + "\n", + " # TODO: Aug 27, 2024: For now, let's jsut override the \"version\" column and fix it when we\n", + " # reconsile the fim_version and model_version columns\n", + " df['version'] = PUBLIC_FIM_VERSION\n", + " df[COLUMN_NAME_FIM_VERSION] = PUBLIC_FIM_VERSION\n", + " df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", + "\n", + " # --------------------------------------\n", + " # Load to DB\n", + " # Chunk load data into DB\n", + " if catfim_type in ['flow_based_catfim', 'stage_based_catfim']:\n", + "\n", + " # Create list of df chunks\n", + " n = 1000 # chunk row size\n", + " print(f\"Chunk loading... into {table_name} -- {n} records at a time\")\n", + " print(\"\")\n", + " chunk_df = [df[i:i+n] for i in range(0, df.shape[0], n)]\n", + "\n", + " # Load the first chunk into the DB as a new table\n", + " first_chunk_df = chunk_df[0]\n", + " num_chunks = len(chunk_df)\n", + "\n", + " print(f\" ... loading chunk 1 of {num_chunks}\")\n", + "\n", + " first_chunk_df.to_sql(\n", + " name=table_name,\n", + " con=db_engine,\n", + " schema='reference',\n", + " if_exists='replace',\n", + " index=False,\n", + " dtype={'oid': sqlalchemy.types.Integer(),\n", + " 'geom': Geometry('MULTIPOLYGON', srid=src_crs)}\n", + " )\n", + "\n", + " # Load remaining chunks into newly created table\n", + " ctr = 1 # Already loaded one\n", + " for remaining_chunk in chunk_df[1:]:\n", + " # print(remaining_chunk.shape[0])\n", + " ctr += 1\n", + " print(f\" ... loading chunk {ctr} of {num_chunks}\")\n", + " remaining_chunk.to_sql(\n", + " name=table_name,\n", + " con=db_engine,\n", + " schema='reference',\n", + " if_exists='append',\n", + " index=False,\n", + " dtype={'oid': sqlalchemy.types.Integer(),\n", + " 'geom': Geometry('MULTIPOLYGON', srid=src_crs)\n", + " }\n", + " )\n", + " # end for\n", + " else: # sites tables\n", + " print(f\"Loading data into {table_name} ...\")\n", + "\n", + " df.to_sql(\n", + " name=table_name,\n", + " con=db_engine,\n", + " schema='reference',\n", + " if_exists='replace',\n", + " index=False,\n", + " dtype={'oid': sqlalchemy.types.Integer(),\n", + " 'geom': Geometry('POINT', srid=src_crs)}\n", + " )\n", + "\n", + " # This should auto create a gist index against the geometry column\n", + " # if that index name already exists, the upload will fail, the index can not pre-exist\n", + " # Best to drop the table before loading.\n", + "\n", + " # return\n", + "\n", + "print(\"load_catfim_table function loaded\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "e2b43cdc-8591-47f4-8db7-1058af5863f5", + "metadata": { + "tags": [] + }, + "source": [ + "

13.a - Backup old DBs and prepare new databases (but not the \"public\" FIM 10/30 db's)

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9577edb-4aa6-423b-819e-df8c922c7ec2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# This covers both Stage Based and Flow Based (but not the \"public\" catfim db's)\n", + "\n", + "# The \"Public\" db backups ana loads are in cells lower (12.d and higher)\n", + "\n", + "# DONE for 4.4.0.0. (4.5.2.11)\n", + "\n", + "# # print(\"Starting Data Backups and table drops for stage and flow based catfim\")\n", + "# db_names = [\"stage_based_catfim\", \"stage_based_catfim_sites\",\n", + "# \"flow_based_catfim\", \"flow_based_catfim_sites\"]\n", + "\n", + "# for db_name in db_names:\n", + "# new_table_name = f\"reference.{db_name}_{OLD_FIM_TAG}\"\n", + "# sql = f'''\n", + "# CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.{db_name};\n", + "# '''\n", + "# sf.execute_sql(sql, db_type='egis')\n", + "# print(f\"{db_name} copied to {new_table_name} if it does not already exist\")\n", + "\n", + "\n", + "# Aug 2024: Now we can drop the tables as we don't have any indexes on them at this time other than the gist geom index.\n", + "# By dropping them, we can auto adjust the tables schema. (don't truncate)\n", + "\n", + "# for db_name in db_names:\n", + "# sf.execute_sql(f\"DROP TABLE IF EXISTS reference.{db_name};\", db_type='egis')\n", + "# print(f\"reference.{db_name} table dropped if it existed\")\n", + "\n", + "\n", + "# print(\"Data Backups of flow based catfim are complete\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "4377dc64-963e-447e-a553-1a59c7cb1781", + "metadata": { + "tags": [] + }, + "source": [ + "

13.b - Updated Flow and Stage Based CatFIM Data (Non Public)

\n", + "\n", + "

AUG 2024: IMPORTANT NOTE:

\n", + "The stage based catfim (library) csv has grown to appx 10 GiB. Our current notebook, hv-vpp-ti-viz-notebook only has 15 GiB memory.\n", + "Running tool can easily overwhelm the notebook server and freeze it up forcing a reboot.\n", + "Sometimes when the notebook instance comes back up, it no longer has ths swap system in place. You will need most of the memory\n", + "and some swap to load it. Keep an eye a \"terminal\" windows and keep entering `free -h` to keep an eye on it's usage.\n", + "
\n", + "We will need to review to see if we want to:\n", + "\n", + "1. Upgrade this notebook server with more memory (and harddrive space would be good)\n", + "\n", + "2. Change the load of the catfim library (non sites) data to another system. Maybe we can load it via a lambda to an EC2 or something?\n", + "\n", + "3. Get the FIM Team to break it to smaller pieces, but watch carefully for the OID system (unique id for all records)\n", + "\n", + "**When you are done running this script, Please restart this kernal as it does not appear to be releasing all memory. (memory leak?)**\n", + "\n", + "\n", + "Also looks like Tyler has some notebooks where he was moving this into a lambda load? We need to look into that\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95c55cd0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "print(\"Starting of CatFIM data\")\n", + "\n", + "# catfim_types = ['flow_based_catfim', 'flow_based_catfim_sites']\n", + "# catfim_types = ['stage_based_catfim', 'stage_based_catfim_sites']\n", + "catfim_types = ['stage_based_catfim_sites']\n", + "# catfim_types = ['stage_based_catfim']\n", + "\n", + "start_dt = datetime.now()\n", + "\n", + "for catfim_type in catfim_types:\n", + " print(f\"Loading {catfim_type} data\")\n", + " load_catfim_table(catfim_type)\n", + "\n", + "print(\"\")\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "db3eef2c-8764-4c0a-9e4d-07d9bb6d4dfe", + "metadata": { + "tags": [] + }, + "source": [ + "

13.c - CatFIM Backup old \"public\" FIM 10 / 30 DBs and prepare new databases

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5cf108d-7360-48a9-a4a2-81b23b9e51b4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "'''\n", + "This covers ONLY Catfim public FIM 10/30 for both flow based and stage based\n", + "'''\n", + "\n", + "''' DONE for 4.4.0.0. (4.5.2.11)'''\n", + "\n", + "# db_name_appendix = f\"{OLD_FIM_TAG}_fim_10\"\n", + "\n", + "# print(\"Starting Data Backups and table drops for stage and flow based PUBLIC catfim\")\n", + "# # db_names = [\"stage_based_catfim_public\", \"stage_based_catfim_sites_public\",\n", + "# # \"flow_based_catfim_public\", \"flow_based_catfim_sites_public\"]\n", + "\n", + "# # stage_based_catfim_sites_public didn't exist for fim 10 but should have in TI (does in other enviros likely)\n", + "# db_names = [\"stage_based_catfim_public\", \n", + "# \"flow_based_catfim_public\", \"flow_based_catfim_sites_public\"]\n", + "\n", + "# for db_name in db_names:\n", + "# new_table_name = f\"reference.{db_name}_{db_name_appendix}\"\n", + "# sql = f\"CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.{db_name}\"\n", + "# sf.execute_sql(sql, db_type='egis')\n", + "# print(f\"{db_name} copied to {new_table_name} if it does not already exist\")\n", + "\n", + " \n", + "# # Aug 2024: Now we can drop the tables as we don't have any indexes on them at this time other than the gist geom index.\n", + "# # By dropping them, we can auto adjust the tables schema. (don't truncate)\n", + "\n", + "# for db_name in db_names:\n", + "# sf.execute_sql(f\"DROP TABLE IF EXISTS reference.{db_name};\", db_type='egis')\n", + "# print(f\"reference.{db_name} table dropped if it existed\")\n", + "\n", + "# print(\"Data Backups of flow based catfim are complete\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "3e451ed4-148f-4ccd-83bf-dff900915efb", + "metadata": { + "tags": [] + }, + "source": [ + "

13.d - Load CatFIM \"public\" FIM 30 DBs

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd38c00d-22ad-476e-ab05-cf293e5bbc15", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "\n", + "print(\"Loading CatFIM Public datasets (FIM 30)\")\n", + "\n", + "catfim_types = [\"stage_based_catfim\", \"stage_based_catfim_sites\",\n", + " \"flow_based_catfim\", \"flow_based_catfim_sites\"]\n", + "\n", + "__public_fim_release = \"fim_30\" # The new fim public release being loaded (ie. fim_10, fim_30, fim_60..)\n", + "\n", + "start_dt = datetime.now()\n", + "\n", + "for catfim_type in catfim_types:\n", + " print(\"\")\n", + " sql = f'''\n", + " DROP TABLE IF EXISTS reference.{catfim_type}_public;\n", + "\n", + " SELECT\n", + " catfim.*,\n", + " '{__public_fim_release}' as public_fim_release\n", + " INTO reference.{catfim_type}_public\n", + " FROM reference.{catfim_type} as catfim\n", + " JOIN reference.public_fim_domain as fim_domain ON ST_Intersects(catfim.geom, fim_domain.geom)\n", + " '''\n", + " print(sf.execute_sql(sql, db_type='egis'))\n", + " print(f\"public {__public_fim_release} data load for {catfim_type} is complete\")\n", + "\n", + "# what about indexes again?\n", + "\n", + "# for db_name in db_names:\n", + "# new_table_name = f\"reference.{db_name}_{db_name_appendix}\"\n", + "# sql = f\"CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.{db_name}\"\n", + "# sf.execute_sql(sql, db_type='egis')\n", + "# print(f\"{db_name} copied to {new_table_name} if it does not already exist\")\n", + "\n", + "print(\"\")\n", + "end_dt = datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "65578391-f29e-4de4-8d98-ab48a0375603", + "metadata": { + "tags": [] + }, + "source": [ + "

14 - Clear the HAND Cache

" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3d1ab20-4148-4d1d-bbb6-a2c3983ba145", + "metadata": {}, + "outputs": [], + "source": [ + "sql = \"\"\"\n", + "TRUNCATE TABLE fim_cache.hand_hydrotable_cached;\n", + "TRUNCATE TABLE fim_cache.hand_hydrotable_cached_max;\n", + "TRUNCATE TABLE fim_cache.hand_hydrotable_cached_geo;\n", + "TRUNCATE TABLE fim_cache.hand_hydrotable_cached_zero_stage;\n", + "\"\"\"\n", + "sf.execute_sql(sql)" + ] + }, + { + "cell_type": "markdown", + "id": "8af909c1-d6a6-4ce7-85b5-080d0a556c90", + "metadata": {}, + "source": [ + "

15 - SAVE TO REPO (AND REDEPLOY TO TI WITH NEW VERSION VARIABLE IN TERRAFORM ??)

\n", + "\n", + "TODO: Aug 2024: Come up with system to save changes to this script and related scripts\n", + "Note from Rob: While, un-elegant, there so much quick evolution here that I recommend we even keep seperate named load scripts in GIT\n", + "ie) one for FIM Version 4.4.0.0 and one for 4.5.2.11, etc. So many changes for each edition and very fast script changes WIP may \n", + "make it smarter to keep each script seperately (ie. 4.4.0.0, 4.5.2.11, etc)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "conda_python3", + "language": "python", + "name": "conda_python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From ba51b48e5685baf15c0ab4ef4e07e7209222d4d6 Mon Sep 17 00:00:00 2001 From: RobH Date: Mon, 21 Oct 2024 23:18:09 +0000 Subject: [PATCH 02/18] Update basic overall steps for new load --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 717 +++++------------- 1 file changed, 169 insertions(+), 548 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index de767307..c201f6be 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "markdown", - "id": "eae54700-98da-48c9-95d3-97dde219b8d5", + "id": "d2741e09-a2f3-409c-adc1-bf6f28e18646", "metadata": { "tags": [] }, @@ -31,8 +31,7 @@ "2. `Lambda FIM_PREFIX` : \n", "3. `Lambda FIM_VERSION and Memory` : \n", "4. `ras2fim` : No update in this release. Keep the last load.\n", - "5. `ras2fim Boundaries`: No update required for this release.\n", - "6. `AEP`\n", + "5. `AEP`\n", " - `2 year` : -- \n", " - `5 year` : -- \n", " - `10 year` : -- \n", @@ -40,20 +39,20 @@ " - `50 year` : -- \n", " - `HW / High Water` : -- \n", " - `Change the hv-vpp-ti-viz-fim-data-prep Lambda memory back to 2048mb` : -- \n", - "7. `Catchments`\n", + "6. `Catchments`\n", " - `Branch 0` : -- \n", " - `GMS` : -- \n", - "8. `usgs_elev_table` : -- \n", - "9. `hydrotable / hydrotable_staggered` : -- \n", - "10. `usgs_rating_curve / usgs_rating_curves staggered` : -- \n", - "11. `Skills Metrics` : -- \n", - "12. `FIM Performance` : -- \n", - "13. `CatFIM`\n", + "7. `usgs_elev_table` : -- \n", + "8. `hydrotable / hydrotable_staggered` : -- \n", + "9. `usgs_rating_curve / usgs_rating_curves staggered` : -- \n", + "10. `Skills Metrics` : -- \n", + "11. `FIM Performance` : -- \n", + "12. `CatFIM`\n", " - `Stage Based CatFIM` : -- \n", " - `Flow Based CatFIM` : -- \n", " - `CatFIM FIM 30` : Stage based only? flow not needed but confirm this.\n", - "14. `Clear HAND cache` :\n", - "15. `GIT1 and `Terraform ??` : TBD\n", + "13. `Clear HAND cache` :\n", + "14. `GIT` and `Terraform ??` : We can now do GitHub check in from here. Watch for branches.\n", "\n" ] }, @@ -399,25 +398,19 @@ }, { "cell_type": "markdown", - "id": "0ef11c15-c1c1-46a8-a65a-fa2a0c1dc97b", + "id": "840db921-0e13-4057-8e5a-1dd45f33d614", "metadata": { "tags": [] }, "source": [ - "

4 - LOAD AND UPDATE RAS2FIM DATA IN DB

\n", + "

4 - UPDATE RAS2FIM DATA (inc ras2fim boundaries) IN DB

\n", "\n", - "

Note about ras2fim domain extents

\n", - "As of Aug 2024, a new service came online for a new layer for ras2fim domain extents. Don took care of it.\n", - "The new extent data was loaded as part of different tools and processes, but we will likly want to consolidate\n", - "it to here.\n", + "As of Oct 2024, we have a new fim (hand) release covered in this file, but ras2fim does not have a new\n", + "release. ras2fim will likely be loaded as new datasets become available. The code for ras2fim is removed\n", + "here from the 4.5.2.11 set and will rebuilt as it''s own new separate load script when that happens.\n", "\n", - "When ras2fim datasets are released, they come with a \"release\" package that has all of the ras2fim models and geocurves\n", - "needed here, but also has domain extents for each HUC in the release package. That entire thing is loaded to S3\n", - "for HV to load. As we will upload a new ras2fim data / geocurves and domain extents at the same time, those load\n", - "scripts should all stay together (here for now). We can add that next time.\n", - "\n", - "However.. ras2fim will likely do releases a lot more regularily than FIM, so it should get it's own independeant load scripts\n", - "which this script can optionally reference if it likes (well.. future versions of this script, ie FIM 4.8.x.x or whatever)" + "However, we will have a few modifications for ras2fim data (not a reload) to help bring in the new\n", + "fim_version and model_version columns. Those changes are included here.\n" ] }, { @@ -449,507 +442,21 @@ "# new_table_name = f\"{tbl_name}_{OLD_FIM_TAG}\"\n", "# sql = f\"CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE {tbl_name};\"\n", "# sf.execute_sql(sql)\n", - "# print(f\"{tbl_name} copied to {new_table_name} if it does not already exists\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e8c3939-f2fc-42de-921d-423d69e023f1", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "\n", - "# NOTE: This can be removed in future ras2fim loads.\n", - "\n", - "\n", - "# Temp Aug 2024: We originally just did a table rename for the {table name} to add _4_4_0_0 on it.\n", - "# Then discovered that renaming it means the indexes are now with the new renamed tables\n", - "# When we load the ras2fim tables, they can't have some of the indexes in place.\n", - "# So.. for now, we are going to rename the _4_4_0_0 tables back to their original name, the do the backup\n", - "# above.\n", - "#sf.execute_sql(f'ALTER TABLE IF EXISTS ras2fim.geocurves_4_4_0_0 RENAME TO geocurves;')\n", - "#sf.execute_sql(f'ALTER TABLE IF EXISTS ras2fim.max_geocurves_4_4_0_0 RENAME TO max_geocurves')\n", - "#print(\"Done renaming them back\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60f910bf-4da7-4e62-b9b6-e981f21a23af", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "\n", - "# This function is not efficient, but as ras2fim has a built in geometry columns, it loads it as a string and not a \"geometry\" object.\n", - "# we have to add the records one at a time.\n", - "\n", - "# Aug 2024: Maybe eventually I can make this more generic, but for now it is ras2fim specific\n", - "# see the new one for catfim as it can likely be rolled into one function\n", - "\n", - "# UPDATE: Sep 19, 2024: We had to remove the chunking portion as we discovered that each csv being loaded might have \n", - "# different crs's. You have to know the incoming crs in order to reproject as the incoming csv's can not be used\n", - "# to auto detect the crs. We put in cards for the ras2fim team to have all final csv's come out as a standard\n", - "# projection (perferraly 3857). Going back to chunking will slow down our DB writes and speed it back up again\n", - "\n", - "# Most of the temp comment code is still in place for chunking.\n", - "\n", - "\n", - "def load_ras2fim_files_into_db(csv_file_list, s3_source_parent_prefix, schema_name, db_name):\n", - "\n", - " # TODO: change these to params and make more generic\n", - " # also tell that this is only if you have a geometry column (for now)\n", - "\n", - " print(f\"Loading data to database {schema_name}.{db_name}\")\n", - " print(\"\")\n", - "\n", - " if len(csv_file_list) == 0:\n", - " raise Exception(\"csv file list is empty\")\n", - "\n", - "\n", - " # source_crs = \"epsg:2277\" # (it is coming in as 5070) but we are changing it to 3857 as loading\n", - "\n", - " # The server has limited memory but it is faster to load as many csv's in at a time\n", - " # as resonablty possible. We are going to try it at chunks of 50 (50 csv files) which for ras2fim\n", - " # shouls be appx 2,000 records, but for ras2fim V2, we have 750 (ish) files.\n", - "\n", - " # We can leave this open the entire times as well.\n", - " s3_client = boto3.client(\"s3\")\n", - " default_kwargs = {\"Bucket\": FIM_BUCKET, \"Prefix\": s3_source_parent_prefix}\n", - "\n", - "\n", - " # chunk_size = 25 # number of csv's to load per set\n", - " total_row_count = 0 # all csv row counts combined. You should see this as a record count in the db when done\n", - " r2f_df = None # a re-used concatenating pd dataframe loading up sets of 20 csvs\n", - " # is_new_df = True # After we db load a set, we reset this to start a new set\n", - " is_first_db_set = True # Very first db load\n", - "\n", - " num_recs = len(csv_file_list)\n", - " print(f\"Total number of files to process are {num_recs}\")\n", - "\n", - " # We are going ot keep the db connection open the entire time. \n", - " # It is slow to open/close connections\n", - " # It \"should not\" ?? block any other scripts / services from usign it\n", - " # most Sql servers allow for more than one connection at a time.\n", - " db = database(db_type=\"viz\")\n", - "\n", - " for idx, full_file_url in enumerate(csv_file_list):\n", - "\n", - " print(f\"Dowloading {idx + 1} of {num_recs} files: {full_file_url}\")\n", - "\n", - " is_first_db_set = idx == 0\n", - " # if idx > 4:\n", - " # return # stub test\n", - "\n", - "# if is_new_df is True:\n", - " s3_client = boto3.client(\"s3\")\n", - " default_kwargs = {\"Bucket\": FIM_BUCKET, \"Prefix\": s3_source_parent_prefix}\n", - "\n", - "\n", - " # is_new_df = False\n", - " # else:\n", - " # temp_df = pd.read_csv(full_file_url)\n", - " # total_row_count += len(temp_df)\n", - " # r2f_df = pd.concat([r2f_df, temp_df])\n", - "\n", - " # we want it merge into the db on each xth (chunk size) record or the last record\n", - " # if ((idx + 1) == num_recs) or ((idx+1) % chunk_size == 0):\n", - "\n", - " # download the csv via pandas into a dataframe\n", - " r2f_df = pd.read_csv(full_file_url)\n", - "\n", - " total_row_count += len(r2f_df)\n", - " r2f_df = r2f_df.fillna(0)\n", - "\n", - " cur_csv = r2f_df.loc[0, 'crs']\n", - " # print(f\"Original crs = {cur_csv}\")\n", - "\n", - " # Create a new source_unit_id which traces back to the folder and code to create\n", - " # this specific huc and model in ras2fim\n", - " r2f_df['source_unit_id'] = r2f_df.apply(lambda row: row.unit_name + \"_\" + \n", - " str(row.unit_version), axis=1)\n", - " r2f_df.rename(columns={'source_code': 'feature_id_source_code', 'geometry': 'geom'}, inplace=True)\n", - " r2f_df['geom'] = r2f_df['geom'].apply(wkt.loads)\n", - "\n", - " # print(f\"... Next set of downloads and adjustments complete, now to db load - Last Idx: {idx + 1} \")\n", - "\n", - " r2f_geodf = gpd.GeoDataFrame(data=r2f_df, geometry='geom', crs=cur_csv)\n", - " # print(r2f_geodf)\n", - " # print(\"\")\n", - " r2f_reproj = r2f_geodf.to_crs(\"epsg:3857\")\n", - "\n", - " # If this is the first load, the type must be the value of \"replace\", else \"append\"\n", - " load_type = 'replace' if is_first_db_set is True else 'append'\n", - "\n", - " r2f_reproj.to_postgis(\n", - " name=db_name,\n", - " con=VIZ_DB_ENGINE,\n", - " schema=schema_name,\n", - " if_exists=load_type,\n", - " index=False,\n", - " )\n", - " print(\"... db load complete\")\n", - "\n", - " # Sanity check on crs\n", - " # if is_first_db_set:\n", - " # print(sf.run_sql_in_db(f\"SELECT ST_SRID(geom) FROM {schema_name}.{db_name} LIMIT 1\"))\n", - "\n", - " r2f_df = None\n", - " r2f_geodf = None\n", - " r2f_reproj = None\n", - " # is_new_df = True # reset it for the next set\n", - " s3_client = None # resets it so it is not open so long. It timse out if open too long\n", - " is_first_db_set = False\n", - "\n", - " # break\n", - "\n", - " # else don't write to db tu continue on to the next record\n", - "\n", - " # end for\n", - " print(\"\")\n", - " print(\"--------------------------------------------------------------\")\n", - " print(\"All records now loaded to the database\")\n", - "\n", - " with db.get_db_connection() as conn, conn.cursor() as cur:\n", - " # after all records are loaded to the db.\n", - " print(f\"Adding {COLUMN_NAME_FIM_VERSION} column to {schema_name}.{db_name}\")\n", - " sql = f\"ALTER TABLE {schema_name}.{db_name} ADD COLUMN IF NOT EXISTS {COLUMN_NAME_FIM_VERSION} text DEFAULT '{PUBLIC_FIM_VERSION}';\"\n", - " cur.execute(sql)\n", - " conn.commit()\n", - "\n", - " print(f\"Adding {COLUMN_NAME_MODEL_VERSION} column to {schema_name}.{db_name}\")\n", - " sql = f\"ALTER TABLE {schema_name}.{db_name} ADD COLUMN IF NOT EXISTS {COLUMN_NAME_MODEL_VERSION} text DEFAULT '{model_version}';\"\n", - " cur.execute(sql)\n", - " conn.commit()\n", - "\n", - " print(\"Dropping un-necessary columns from DB ...\")\n", - " drop_col_names = [\"profile_num\", \"model_id\", \"xs_us\", \"xs_ds\", \"unit_name\", \"unit_version\", \"version\", \"crs\"]\n", - " # print(drop_col_names)\n", - " # print(\"\")\n", - "\n", - " sql = f\"ALTER TABLE {schema_name}.{db_name} \"\n", - " for col_name in drop_col_names:\n", - " sql += f\" DROP COLUMN {col_name},\"\n", - "\n", - " # the last char is a comma and we need to change it to be \" CASCASE;\"\n", - " sql = sql[0:-1] + \" CASCADE;\"\n", - " print(sql)\n", - " cur.execute(sql)\n", - " conn.commit()\n", - " print(f\"Total Rows loaded to DB is {total_row_count}\")\n", - " # end of def\n", - "\n", - "print(\"Download and db load ras2fim function loaded\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e57d7bd1-c852-416e-b259-715b023bf743", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Load the ras2fim.geocurves\n", - "\n", - "# Note: For Aug 2024 (ras 2.0.1.0 with appx 11 hucs, this took appx 1 hr 15 mins to run\n", - "\n", - "model_version = \"ras2fim 2.0.1.0\"\n", - "new_s3_version_folder = \"v2_0\"\n", - "s3_source_parent_prefix = f\"ras2fim/{new_s3_version_folder}\"\n", - "\n", - "start_dt = datetime.now()\n", - "print(\"\")\n", - "print(\"Starting loading of ras2fim.geocurves\")\n", - "\n", - "\n", - "# Aug 21 2024, AWS Creds expired and died just after loading rec 2475 of 7948.\n", - "# Commented out truncate, reset csv_list to be recs 2476 and higher and restarted.\n", - "# All over exact time lost, but can esimate it.\n", - "sql = '''\n", - " TRUNCATE TABLE ras2fim.geocurves;\n", - " TRUNCATE TABLE ras2fim.max_geocurves;\n", - "'''\n", - "print(sf.execute_sql(sql))\n", - "print(\"geocurves and max_geocurves tables truncated to start clean\")\n", - "print(\"\")\n", - "\n", - "# Now download the s3 geocurves\n", - "# Overloaded the server as the memory couldn't handle it.\n", - "# r2f_df = s3_sf.download_S3_csv_files_to_df(FIM_BUCKET, s3_source_parent_prefix, True)\n", - "\n", - "# lets just get a list of files, then iterate over them to load each to the db one at a time.\n", - "r2f_file_names = s3_sf.get_s3_subfolder_file_names(FIM_BUCKET, s3_source_parent_prefix, False)\n", - "\n", - "if len(r2f_file_names) == 0:\n", - " raise Exception(\"No file names found\")\n", - "\n", - "\n", - "csv_file_list = list(filter(lambda x: (x.endswith(\".csv\") == True), r2f_file_names))\n", - "if len(csv_file_list) == 0:\n", - " raise Exception(\"No csv file names found\")\n", - "\n", - "# print(csv_file_list)\n", - "\n", - "# Test against just 20 records for a timing test\n", - "# test_list = csv_file_list[:20]\n", - "# print(test_list)\n", - "\n", - "print(\"Loading df into ras2fim geocurve db\")\n", - "\n", - "load_ras2fim_files_into_db(csv_file_list, s3_source_parent_prefix, 'ras2fim', 'geocurves')\n", - "\n", - "# See note above about having to restart at rec num 2475 (our index displays were 1 based and not zero based\n", - "# restart_list = csv_file_list[974:]\n", - "# load_ras2fim_files_into_db(restart_list, s3_source_parent_prefix, 'ras2fim', 'geocurves')\n", - "\n", - "end_dt = datetime.now()\n", - "time_duration = end_dt - start_dt\n", - "print(\".... ras2fim files now loaded to ras2fim.geocurves\")\n", - "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ded5372c-d94e-4276-be77-b13a94db55cc", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# ras2fim \"previous\" columns loading\n", - "\n", - "print(\"... Starting ras2fim previous stage adding and max_geocurves creating\")\n", - "start_dt = datetime.now()\n", - "\n", - "sql = \"\"\"\n", - "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS previous_stage_ft double precision;\n", - "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS previous_stage_m double precision;\n", - "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS previous_discharge_cfs double precision;\n", - "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS previous_discharge_cms double precision;\n", - "ALTER TABLE IF EXISTS ras2fim.geocurves ADD COLUMN IF NOT EXISTS oid INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY;\n", - "\"\"\" \n", - "print(sf.execute_sql(sql))\n", - "\n", - "sql = '''DROP TABLE IF EXISTS ras2fim.temp_ras2fim_lagged;'''\n", - "print(sf.execute_sql(sql))\n", - "\n", - "\n", - "db = database(db_type=\"viz\")\n", - "with db.get_db_connection() as conn, conn.cursor() as cur:\n", - "\n", - " # PS. It is ok that there are some nulls in the four \"previous\" columns\n", - " sql = \"\"\"\n", - " CREATE TABLE ras2fim.temp_ras2fim_lagged as (SELECT\n", - " feature_id,\n", - " stage_ft,\n", - " (lag(stage_m, 1) OVER (PARTITION BY feature_id ORDER by stage_m)) as previous_stage_m,\n", - " (lag(stage_ft, 1) OVER (PARTITION BY feature_id ORDER by stage_ft)) as previous_stage_ft,\n", - " (lag(discharge_cfs, 1) OVER (PARTITION BY feature_id ORDER by discharge_cfs)) as previous_discharge_cfs,\n", - " (lag(discharge_cms, 1) OVER (PARTITION BY feature_id ORDER by discharge_cms)) as previous_discharge_cms\n", - " FROM ras2fim.geocurves)\n", - " \"\"\"\n", - " cur.execute(sql)\n", - " conn.commit()\n", - "\n", - " sql = \"\"\"\n", - " UPDATE ras2fim.geocurves gc\n", - " SET previous_stage_ft = lagged.previous_stage_ft,\n", - " previous_stage_m = lagged.previous_stage_m,\n", - " previous_discharge_cfs = lagged.previous_discharge_cfs,\n", - " previous_discharge_cms = lagged.previous_discharge_cms\n", - " FROM ras2fim.temp_ras2fim_lagged as lagged\n", - " WHERE gc.feature_id = lagged.feature_id\n", - " and gc.stage_ft = lagged.stage_ft;\n", - " \"\"\"\n", - " cur.execute(sql)\n", - " conn.commit()\n", - "\n", - "\n", - "print(\"Removing ras2fim.temp_ras2fim_lagged table\")\n", - "sql = \"DROP TABLE IF EXISTS ras2fim.temp_ras2fim_lagged;\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "\n", - "print(\"Adding indexes if required\")\n", - "\n", - "sql = \"ALTER TABLE IF EXISTS ras2fim.geocurves OWNER to viz_proc_admin_rw_user;\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "sql = \"ALTER TABLE IF EXISTS ras2fim.geocurves OWNER to viz_proc_admin_rw_user;\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "sql = \"DROP INDEX IF EXISTS ras2fim.geocurves_discharge_cms_index;\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "\n", - "sql = \"\"\"\n", - "CREATE INDEX IF NOT EXISTS geocurves_discharge_cms_index ON ras2fim.geocurves USING btree (discharge_cms ASC NULLS LAST)\n", - "\"\"\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "\n", - "sql = \"DROP INDEX IF EXISTS ras2fim.geocurves_feature_id_index;\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "\n", - "sql = \"\"\"\n", - "CREATE INDEX IF NOT EXISTS geocurves_feature_id_index ON ras2fim.geocurves USING btree (feature_id ASC NULLS LAST)\n", - "\"\"\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "\n", - "sql = \"DROP INDEX IF EXISTS ras2fim.geocurves_previous_discharge_cms_index;\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "\n", - "sql = \"\"\"\n", - "CREATE INDEX IF NOT EXISTS geocurves_previous_discharge_cms_index\n", - " ON ras2fim.geocurves USING btree (previous_discharge_cms ASC NULLS LAST)\n", - "\"\"\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "\n", - "# Skip for now.. not sure if it will be needed in the next set\n", - "# -- DROP INDEX IF EXISTS ras2fim.idx_geocurves_geom;\n", - "# CREATE INDEX IF NOT EXISTS idx_geocurves_geom\n", - "#     ON ras2fim.geocurves USING gist\n", - "#     (geom)\n", - "#     TABLESPACE pg_default;\n", - "\n", - "\n", - "end_dt = datetime.now()\n", - "time_duration = end_dt - start_dt\n", - "print(\".... Done - ras2fim previous stage columns added\")\n", - "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a1bd997c-f813-49dd-94e0-adf062f3b74b", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# ras2fim max_geocurve loading\n", - "\n", - "start_dt = datetime.now()\n", - "\n", - "print(\"Start of creating and loading max_geocurves table\")\n", - "\n", - "# Table can't have any indexes as nothing in unique enough\n", - "# We shoudl have an oid column though\n", - "sql = \"DROP TABLE IF EXISTS ras2fim.max_geocurves;\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "sql = \"\"\"\n", - "CREATE TABLE ras2fim.max_geocurves as (\n", - " SELECT\n", - " feature_id,\n", - " max(discharge_cfs) as max_rc_discharge_cfs,\n", - " max(stage_ft) as max_rc_stage_ft,\n", - " max(discharge_cms) as max_rc_discharge_cms,\n", - " max(stage_m) as max_rc_stage_m\n", - " FROM ras2fim.geocurves\n", - " GROUP BY feature_id )\n", - "\"\"\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "sql = \"DROP INDEX IF EXISTS ras2fim.max_geocurves_feature_id_index;\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "sql = \"\"\"\n", - "CREATE INDEX IF NOT EXISTS max_geocurves_feature_id_index ON \n", - " ras2fim.max_geocurves USING btree (feature_id ASC NULLS LAST);\n", - "\"\"\"\n", - "print(sf.execute_sql(sql))\n", - "\n", - "print(\"max_geocurves table created and filled\")\n", - "\n", - "\n", - "print(\"\")\n", - "\n", - "end_dt = datetime.now()\n", - "time_duration = end_dt - start_dt\n", - "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "bec1b8bf-e1dc-4525-9066-afef7273e289", - "metadata": { - "tags": [] - }, - "source": [ - "

5 - Load the Ras2Fim boundaries into egis

" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "251905aa-259a-4ef4-9590-dcd31375c624", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Moves data from local shapefile to EGIS\n", - "\n", - "# Importing Modules\n", - "import os\n", - "import sys\n", - "import helper_functions.shared_functions as sf\n", - "import geopandas as gpd\n", - "\n", - "# sys path if needed\n", - "sys.path.append(os.path.join(os.path.abspath(''), '..'))\n", - "\n", - "# Dir location of your data\n", - "DATA_DPATH = r\"/home/ec2-user/SageMaker/Don - Campground/Don - Store\"\n", - "# File location of your shapefile data\n", - "DATASET_DPATH = f\"{DATA_DPATH}/main_huc8.shp\"\n", - "# Check path by printing\n", - "print(DATASET_DPATH)\n", - "\n", - "# Only use when you want to create something new\n", - "gdf = gpd.read_file(DATASET_DPATH, columns='geometry')\n", - "gdf.to_postgis(name=\"boundaries2\", con=sf.get_db_engine(db_type=\"viz\"),\n", - " schema=\"ras2fim\", if_exists=\"replace\")\n", - "\n", - "# add an oid field to your data\n", - "sql = \"\"\"\n", - "ALTER TABLE ras2fim.boundaries ADD COLUMN oid SERIAL PRIMARY KEY;\n", - "\"\"\"\n", - "sf.execute_sql(sql)\n", + "# print(f\"{tbl_name} copied to {new_table_name} if it does not already exists\")\n", "\n", - "# delete a table in egis if you made a mistake or such\n", - "sql = \"\"\"\n", - "DROP TABLE IF EXISTS reference.ras2fim_boundaries;\n", - "\"\"\"\n", - "sf.execute_sql(sql, db_type=\"egis\")\n", "\n", - "# The function move_data_from_viz_to_egis is called because the data needs to \n", - "# be in the egis before publishing to uat or prd" + "# TBD: Oct 21, 2024\n", + "# Do we need to do adjustments to the rasfim data for the new fim_version and model_version tables?\n" ] }, { "cell_type": "markdown", - "id": "316b1321-f570-4f53-aeee-f8798330642f", + "id": "0c5994fa-8fa5-471d-86e9-745d67e79ecf", "metadata": { "tags": [] }, "source": [ - "

6 - Run AEP FIM Pipelines.

\n", + "

5 - Run AEP FIM Pipelines.

\n", "Updated Documentation from Tyler Early 2024: This can be done in a couple of diferent ways.\n", "\n", "1) One option is to use the pipeline_input code created below by Corey to start the AEP pipelines directly from this notebook.
\n", @@ -976,6 +483,9 @@ "execution_count": null, "id": "6a698067", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1037,6 +547,9 @@ "execution_count": null, "id": "f6d6ee69", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1062,6 +575,9 @@ "execution_count": null, "id": "a4f89d9a", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1087,6 +603,9 @@ "execution_count": null, "id": "791d1a8b", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1112,6 +631,9 @@ "execution_count": null, "id": "1bb87128", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1137,6 +659,9 @@ "execution_count": null, "id": "4832e4e0", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1162,6 +687,9 @@ "execution_count": null, "id": "187e83bc-ebbe-4615-a046-e0ef7b09ad3b", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1187,6 +715,9 @@ "cell_type": "markdown", "id": "2c810767-2f5d-46b2-860c-5d5c549f2e2a", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "source": [ @@ -1199,20 +730,20 @@ }, { "cell_type": "markdown", - "id": "48074dbe-ead7-467f-ae68-16c746115efc", + "id": "97fa6c78-efc2-4dbe-ae9a-55b3c4842492", "metadata": {}, "source": [ - "

7 - RUN CATCHMENT WORKFLOWS 2 CONFIGS AT A TIME. CHECK FOR STEP FUNCTION FINISHING BEFORE STARTING NEW ONE

" + "

6 - RUN CATCHMENT WORKFLOWS 2 CONFIGS AT A TIME. CHECK FOR STEP FUNCTION FINISHING BEFORE STARTING NEW ONE

" ] }, { "cell_type": "markdown", - "id": "9b33a346-ee44-4844-9687-627ccd601263", + "id": "b4574e4b-1569-4362-8d3f-42f4319752a1", "metadata": { "tags": [] }, "source": [ - "### 7a - Branch 0 Catchments. Wait until it is done before kicking off the next GMS (Level Path) catchments load a bit lower. ###" + "### 6a - Branch 0 Catchments. Wait until it is done before kicking off the next GMS (Level Path) catchments load a bit lower. ###" ] }, { @@ -1220,6 +751,9 @@ "execution_count": null, "id": "c77d6e5b-f5fb-422f-a6c3-0f53743cd631", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1246,6 +780,9 @@ "execution_count": null, "id": "a7495759", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1357,10 +894,10 @@ }, { "cell_type": "markdown", - "id": "d792a88e-51fd-4e28-9360-b90abd99ddaf", + "id": "2c75c4b7-bf13-4e8c-a33e-261219ce4338", "metadata": {}, "source": [ - "### 7b - GMS (Level Paths / non branch 0) catchments ###" + "### 6b - GMS (Level Paths / non branch 0) catchments ###" ] }, { @@ -1368,6 +905,9 @@ "execution_count": null, "id": "3ea2a5be-9936-4abe-832a-8582f3c3af64", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1390,6 +930,9 @@ "execution_count": null, "id": "9ae8917f", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1501,12 +1044,12 @@ }, { "cell_type": "markdown", - "id": "6ff4dc6b-c41b-412c-9d9f-d812c5c72115", + "id": "05fb7850-5be1-4127-86ba-bbfde273424a", "metadata": { "tags": [] }, "source": [ - "

8 - Recreate derived.usgs_elev_table

" + "

7 - Recreate derived.usgs_elev_table

" ] }, { @@ -1530,6 +1073,9 @@ "execution_count": null, "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", "metadata": { + "jupyter": { + "source_hidden": true + }, "scrolled": true, "tags": [] }, @@ -1586,10 +1132,10 @@ }, { "cell_type": "markdown", - "id": "92a02c35-16a9-4cd8-9fd6-61d20b3e6feb", + "id": "9951d778-5d75-4550-a63d-1dee833417f9", "metadata": {}, "source": [ - "

9 - Recreate derived.hydrotable_staggered

" + "

8 - Recreate derived.hydrotable_staggered

" ] }, { @@ -1597,6 +1143,9 @@ "execution_count": null, "id": "18a2a7c2-7499-4ebc-a878-769e62dfbd6b", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1614,6 +1163,9 @@ "execution_count": null, "id": "33026eb6-fbc7-47f7-a44b-f9cea8f0911b", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1689,6 +1241,9 @@ "execution_count": null, "id": "5cc764e4-7e67-4b51-a100-6021e31416cd", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1725,6 +1280,9 @@ "execution_count": null, "id": "74f21c03-ca36-402c-a69b-396162720f66", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1737,12 +1295,12 @@ }, { "cell_type": "markdown", - "id": "0237f374-5829-4050-b503-958c9d4c3703", + "id": "c16ad30a-f0d6-4fba-8a9f-059d3d11ff6a", "metadata": { "tags": [] }, "source": [ - "

10 - Recreate derived.usgs_rating_curves_staggered

" + "

9 - Recreate derived.usgs_rating_curves_staggered

" ] }, { @@ -1750,6 +1308,9 @@ "execution_count": null, "id": "461f20b4-8442-4c0e-9114-ea62764cd789", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1770,6 +1331,9 @@ "execution_count": null, "id": "b752fce5-c8d7-4064-92d7-842c22d1723e", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1789,6 +1353,9 @@ "execution_count": null, "id": "d6a0cc18-de82-4bd8-bd4d-c3ad021a1dec", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1819,6 +1386,9 @@ "execution_count": null, "id": "4548d5e8-303e-44e0-a8a5-7b1947214b4e", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1850,6 +1420,9 @@ "execution_count": null, "id": "d1a77875-f569-4410-888c-b132714a7c9e", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1862,12 +1435,12 @@ }, { "cell_type": "markdown", - "id": "ee640076-be6f-47f5-b091-f60192502bd0", + "id": "2e926c7a-9d1c-41ea-a9b3-5fc688994157", "metadata": { "tags": [] }, "source": [ - "

11 - UPDATE SRC SKILL METRICS IN DB

" + "

10 - UPDATE SRC SKILL METRICS IN DB

" ] }, { @@ -1875,6 +1448,9 @@ "execution_count": null, "id": "95c472f8-0557-4864-8536-1814f3ac4286", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1918,6 +1494,9 @@ "execution_count": null, "id": "9a249854-b9cd-4ccb-afea-96c35708515d", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1933,6 +1512,9 @@ "execution_count": null, "id": "c24f509a-90df-492a-a0d4-aa9eab8e7041", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1964,6 +1546,9 @@ "execution_count": null, "id": "b5639447-9a5a-4add-9c36-ebd945a5a8d0", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2017,6 +1602,9 @@ "execution_count": null, "id": "0df23345-47a4-490a-b10f-2020e3a2a68e", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2028,12 +1616,12 @@ }, { "cell_type": "markdown", - "id": "cf47b262-d5aa-41e3-a12e-99432f52fbfc", + "id": "118c126e-00b8-4c7f-8a8d-0bb427f413f3", "metadata": { "tags": [] }, "source": [ - "

12 - UPDATE FIM PERFORMANCE METRICS IN DB

" + "

11 - UPDATE FIM PERFORMANCE METRICS IN DB

" ] }, { @@ -2041,6 +1629,9 @@ "execution_count": null, "id": "b291a461-e36f-4049-84b6-dd59cd1c4ad2", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2088,6 +1679,9 @@ "execution_count": null, "id": "f4fcdfab-3936-4a0a-8ab9-605e9a64f9bd", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2117,6 +1711,9 @@ "execution_count": null, "id": "06ca5581", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2268,12 +1865,12 @@ }, { "cell_type": "markdown", - "id": "a52c1563-3c91-4810-96d0-48568b60cbef", + "id": "b9309a4b-1d94-4af2-90dc-57a08c4add68", "metadata": { "tags": [] }, "source": [ - "

13 - CatFIM (Stage-Based and Flow-Based)

" + "

12 - CatFIM (Stage-Based and Flow-Based)

" ] }, { @@ -2289,6 +1886,9 @@ "execution_count": null, "id": "37f37c81-d105-4c38-aa49-b4aef40a7543", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2477,12 +2077,12 @@ }, { "cell_type": "markdown", - "id": "e2b43cdc-8591-47f4-8db7-1058af5863f5", + "id": "ca12f850-8826-4177-884a-04f461496d7b", "metadata": { "tags": [] }, "source": [ - "

13.a - Backup old DBs and prepare new databases (but not the \"public\" FIM 10/30 db's)

" + "

12.a - Backup old DBs and prepare new databases (but not the \"public\" FIM 10/30 db's)

" ] }, { @@ -2490,6 +2090,9 @@ "execution_count": null, "id": "f9577edb-4aa6-423b-819e-df8c922c7ec2", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2526,12 +2129,12 @@ }, { "cell_type": "markdown", - "id": "4377dc64-963e-447e-a553-1a59c7cb1781", + "id": "a0af91cf-5a87-4c0b-a7ae-54d6809dd152", "metadata": { "tags": [] }, "source": [ - "

13.b - Updated Flow and Stage Based CatFIM Data (Non Public)

\n", + "

12.b - Updated Flow and Stage Based CatFIM Data (Non Public)

\n", "\n", "

AUG 2024: IMPORTANT NOTE:

\n", "The stage based catfim (library) csv has grown to appx 10 GiB. Our current notebook, hv-vpp-ti-viz-notebook only has 15 GiB memory.\n", @@ -2558,6 +2161,9 @@ "execution_count": null, "id": "95c55cd0", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2585,12 +2191,12 @@ }, { "cell_type": "markdown", - "id": "db3eef2c-8764-4c0a-9e4d-07d9bb6d4dfe", + "id": "51d1a57d-089c-4042-98d8-6893d7a45acf", "metadata": { "tags": [] }, "source": [ - "

13.c - CatFIM Backup old \"public\" FIM 10 / 30 DBs and prepare new databases

" + "

12.c - CatFIM Backup old \"public\" FIM 10 / 30 DBs and prepare new databases

" ] }, { @@ -2598,6 +2204,9 @@ "execution_count": null, "id": "f5cf108d-7360-48a9-a4a2-81b23b9e51b4", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2637,12 +2246,12 @@ }, { "cell_type": "markdown", - "id": "3e451ed4-148f-4ccd-83bf-dff900915efb", + "id": "afb73b36-056f-47b2-90cd-38b26171723a", "metadata": { "tags": [] }, "source": [ - "

13.d - Load CatFIM \"public\" FIM 30 DBs

" + "

12.d - Load CatFIM \"public\" FIM 30 DBs

" ] }, { @@ -2650,6 +2259,9 @@ "execution_count": null, "id": "fd38c00d-22ad-476e-ab05-cf293e5bbc15", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2696,19 +2308,24 @@ }, { "cell_type": "markdown", - "id": "65578391-f29e-4de4-8d98-ab48a0375603", + "id": "eb36b537-f957-457a-9bfa-64d944f85599", "metadata": { "tags": [] }, "source": [ - "

14 - Clear the HAND Cache

" + "

13 - Clear the HAND Cache

" ] }, { "cell_type": "code", "execution_count": null, "id": "a3d1ab20-4148-4d1d-bbb6-a2c3983ba145", - "metadata": {}, + "metadata": { + "jupyter": { + "source_hidden": true + }, + "tags": [] + }, "outputs": [], "source": [ "sql = \"\"\"\n", @@ -2722,12 +2339,16 @@ }, { "cell_type": "markdown", - "id": "8af909c1-d6a6-4ce7-85b5-080d0a556c90", + "id": "b7695cb7-f2f0-4e63-8dae-5e010b5cb308", "metadata": {}, "source": [ - "

15 - SAVE TO REPO (AND REDEPLOY TO TI WITH NEW VERSION VARIABLE IN TERRAFORM ??)

\n", + "

14 - SAVE TO REPO (AND REDEPLOY TO TI WITH NEW VERSION VARIABLE IN TERRAFORM ??)

\n", + "\n", + "Oct 21, 2024: We don''t have a system per-say to update for Terraform, but we now have github hooks\n", + "built right into JupyterHub. We need to figure out how to work with multiple branches and \"getting latest\"\n", + "but this gives us source control management now.\n", + "\n", "\n", - "TODO: Aug 2024: Come up with system to save changes to this script and related scripts\n", "Note from Rob: While, un-elegant, there so much quick evolution here that I recommend we even keep seperate named load scripts in GIT\n", "ie) one for FIM Version 4.4.0.0 and one for 4.5.2.11, etc. So many changes for each edition and very fast script changes WIP may \n", "make it smarter to keep each script seperately (ie. 4.4.0.0, 4.5.2.11, etc)\n" From fd7890e9b9ce567c76e4d163ebc8fcdf1a039bca Mon Sep 17 00:00:00 2001 From: RobH Date: Mon, 21 Oct 2024 23:22:12 +0000 Subject: [PATCH 03/18] Drop Catfim Stage based Sea test --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index c201f6be..27f2c369 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -1886,9 +1886,6 @@ "execution_count": null, "id": "37f37c81-d105-4c38-aa49-b4aef40a7543", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1963,19 +1960,6 @@ " df.loc[df.ahps_lid == nws_lid, 'viz'] = 'no'\n", " df = df[df['viz'] == 'yes']\n", "\n", - " # TODO: Aug 2024: This may be a bug or very outdated. It was in the code to load stage for 4.4.0.0\n", - " # and I left it here for 4.5.2.11, but made a card with the FIM team to review and fix it in there code\n", - " # so we can drop this.\n", - " if 'stage_based' in catfim_type:\n", - " for sea_level_site in ['qutg1', 'augg1', 'baxg1', 'lamf1', 'adlg1', 'hrag1', 'stng1']:\n", - " if \"sites\" in catfim_type:\n", - " df.loc[df.ahps_lid==sea_level_site, 'mapped'] = 'no'\n", - " df.loc[df.ahps_lid==sea_level_site, 'status'] = 'Stage thresholds seem to be based on sea level and not channel thalweg'\n", - " else:\n", - " df.loc[df.ahps_lid==sea_level_site, 'viz'] = 'no'\n", - " df = df[df['viz']=='yes'] # Subset df to only sites desired for mapping\n", - " # end if\n", - "\n", " # Enforce data types on df before loading in DB (TODO: need to create special cases for each layer).\n", " df = df.astype({'huc8': 'str'})\n", " df = df.fillna(0)\n", @@ -2161,9 +2145,6 @@ "execution_count": null, "id": "95c55cd0", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -2339,8 +2320,10 @@ }, { "cell_type": "markdown", - "id": "b7695cb7-f2f0-4e63-8dae-5e010b5cb308", - "metadata": {}, + "id": "166702e8-60d2-485e-a5bc-23b475841e5a", + "metadata": { + "tags": [] + }, "source": [ "

14 - SAVE TO REPO (AND REDEPLOY TO TI WITH NEW VERSION VARIABLE IN TERRAFORM ??)

\n", "\n", @@ -2351,7 +2334,10 @@ "\n", "Note from Rob: While, un-elegant, there so much quick evolution here that I recommend we even keep seperate named load scripts in GIT\n", "ie) one for FIM Version 4.4.0.0 and one for 4.5.2.11, etc. So many changes for each edition and very fast script changes WIP may \n", - "make it smarter to keep each script seperately (ie. 4.4.0.0, 4.5.2.11, etc)\n" + "make it smarter to keep each script seperately (ie. 4.4.0.0, 4.5.2.11, etc)\n", + "\n", + "

Make sure to Publish the changes to git and add a PR

\n", + "\n" ] } ], From a11967a74054666bdb6f56bb85427678dc508553 Mon Sep 17 00:00:00 2001 From: RobH Date: Thu, 31 Oct 2024 20:29:39 +0000 Subject: [PATCH 04/18] checkin before merging in ti --- .gitignore | 6 +- .../viz_db_ingest/lambda_function.py | 20 +++ .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 128 +++++++++++------- 3 files changed, 107 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 083d8970..f1388ae6 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,8 @@ terraform.rc # Ignore all jupyter checkpoint folders .ipynb_checkpoints -Dev/env.yaml \ No newline at end of file +Dev/env.yaml + +# Ignore pycachefiles +**/__pycache__/* +**/__pycache__ diff --git a/Core/LAMBDA/viz_functions/viz_db_ingest/lambda_function.py b/Core/LAMBDA/viz_functions/viz_db_ingest/lambda_function.py index 2941fa3a..b0c29042 100644 --- a/Core/LAMBDA/viz_functions/viz_db_ingest/lambda_function.py +++ b/Core/LAMBDA/viz_functions/viz_db_ingest/lambda_function.py @@ -62,6 +62,26 @@ def lambda_handler(event, context): ds = xr.open_dataset(download_path) ds_vars = [var for var in ds.variables] + match = re.search(r'common/data/model/com/nwm/prod/nwm.\d{8}/(.*)', file) + test_bucket = 'hydrovis-ti-deployment-us-east-1' + s3_fname = f'test_nwm_outputs/{match.group(1)}' + local_fname = f'/tmp/{match.group(1)}' + if 'forcing' in file: + interest_val = 'RAINRATE' + elif 'channel_rt' in file: + interest_val = 'streamflow' + elif 'coastal' in file: + interest_val = 'elevation' + elif 'reservoir' in file: + interest_val = 'water_sfc_elev' + + try: + check_if_file_exists(test_bucket, s3_fname, download=False) + except: + ds['streamflow'] = (ds['streamflow'] + 1) * 1000 + ds.to_netcdf(local_fname) + s3.upload_file(local_fname, test_bucket, s3_fname) + if not target_cols: target_cols = ds_vars diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 27f2c369..ae79b335 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -58,20 +58,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "9e1ee2b5-b109-49e8-8255-f06449b44ee6", "metadata": { "scrolled": true, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: geopandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.0.1)\n", + "Requirement already satisfied: numpy>=1.22 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.1.2)\n", + "Requirement already satisfied: pyogrio>=0.7.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (0.10.0)\n", + "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (24.1)\n", + "Requirement already satisfied: pandas>=1.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.2.2)\n", + "Requirement already satisfied: pyproj>=3.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (3.7.0)\n", + "Requirement already satisfied: shapely>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.0.6)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2.9.0)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.1)\n", + "Requirement already satisfied: certifi in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pyogrio>=0.7.2->geopandas) (2024.7.4)\n", + "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=1.4.0->geopandas) (1.16.0)\n", + "Collecting geoalchemy2\n", + " Downloading GeoAlchemy2-0.15.2-py3-none-any.whl.metadata (2.1 kB)\n", + "Requirement already satisfied: SQLAlchemy>=1.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (2.0.31)\n", + "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (24.1)\n", + "Requirement already satisfied: typing-extensions>=4.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (4.12.2)\n", + "Requirement already satisfied: greenlet!=0.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (3.0.3)\n", + "Downloading GeoAlchemy2-0.15.2-py3-none-any.whl (73 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.6/73.6 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: geoalchemy2\n", + "Successfully installed geoalchemy2-0.15.2\n", + "All loaded\n" + ] + } + ], "source": [ "# Cell to manually pip reload a packages that the Jupyter engine not retained\n", "# !pip install numpy\n", - "# !pip install geopandas\n", + "!pip install geopandas\n", "# !pip install pyarrow\n", "# !pip install xarray\n", - "# !pip install geoalchemy2\n", + "!pip install geoalchemy2\n", "# !pip install contextily\n", "# !pip install rioxarray\n", "print(\"All loaded\")" @@ -91,13 +121,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "5b656259", "metadata": { "scrolled": true, "tags": [] }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "imports loaded\n" + ] + } + ], "source": [ "import os\n", "import codecs\n", @@ -124,13 +174,17 @@ "\n", "sys.path.append(os.path.join(os.path.abspath(''), '..'))\n", "\n", - "import helper_functions.s3_shared_functions as s3_sf\n", - "import helper_functions.shared_functions as sf\n", + "# import helper_functions.s3_shared_functions as s3_sf\n", + "# import helper_functions.shared_functions as sf\n", "# import helper_functions.viz_classes\n", "\n", + "import s3_shared_functions as s3_sf\n", + "import shared_functions as sf\n", + "\n", "from helper_functions.viz_classes import database\n", "from helper_functions.viz_db_ingest import lambda_handler as execute_db_ingest\n", "\n", + "\n", "print(\"imports loaded\")\n", "\n", "# Note: Aug 2024: Sometimes if you need to do the pip install above, you need to reload this twice.. must be a circular dependency ?? or forced pkg reload\n", @@ -139,32 +193,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "e944eff3-6023-48f4-9f57-27304a240447", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Global Variables loaded\n" + ] + } + ], "source": [ "\n", - "# Aug 5, 2024: # The variable named FIM_VERSION will continue to be the field that joins all data together.\n", - "# But we need two new public display fields in the UI. We will no longer show a UI field which previously would have been\n", - "# \"FIM_4_5_2_11\". That won't be displayed anymore. The public field of:\n", - "# public_fim_version, for this edition, becomes \"FIM 5_0_0\" (yes.. three segs)\n", - "# public_model_version, for this edition, becomes \"HAND 4_5_2_11\". \n", - "# When we add ras2fim into the system, it's public_fim_version continues to be FIM 5_0_0,\n", - "# but ras2fim public_model_version, becomes \"ras2fim 2_0_3_0\"\n", + "OLD_MODEL_VERSION = \"HAND 4.5.2.11\"\n", + "OLD_PUBLIC_VERIONS = \"HAND 4.5.2.11\"\n", + "PUBLIC_FIM_VERSION = \"FIM 5.1.0\"\n", + "FIM_MODEL_VERSION = \"HAND 4.5.11.1\"\n", "\n", - "# NOTE: sep 19, 2024: creaing the 4.4.0.0 was just for learning purposes. Now Rob has access to UAT db's so I can compare against\n", - "# those next time if needed. We might remove the references to 4.4.0.0 next time\n", - "\n", - "OLD_FIM_VERSION = \"4.4.0.0\"\n", - "NEW_FIM_VERSION = \"4.5.2.11\"\n", - "PUBLIC_FIM_VERSION = \"FIM 5.0.0\" \n", - "FIM_MODEL_VERSION = \"HAND 4.5.2.11\" # on next major build (after Aug 2024, change this to space and dots. ie) HAND 4.5.2.11)\n", - "OLD_FIM_TAG = OLD_FIM_VERSION.replace('.', '_')\n", - "\n", - "FIM_ROOT_DPATH = f\"fim/fim_{NEW_FIM_VERSION.replace('.', '_')}\"\n", + "FIM_ROOT_DPATH = \"fim/hand_4_5_11_1\"\n", "HAND_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/hand_datasets\"\n", "QA_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/qa_datasets\"\n", "\n", @@ -180,10 +230,9 @@ "# If this is not set correctly, you will get an HTTP error 400 when you call S3 lower.\n", "# You might also see an error of 'An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist.\" the creds are not correct\"\n", "\n", - "# Helps us get to the keys. Note: This was added Oct 16, 2024 and is untested\n", - "sys.path.append(os.path.join(os.path.abspath(''), '../../../../AWS_Secret_keys'))\n", - "import AWS_Keys\n", - "\n", + "# # Helps us get to the keys. Note: This was added Oct 16, 2024 and is untested\n", + "# sys.path.append(os.path.join(os.path.abspath(''), '../../../../AWS_Secret_keys'))\n", + "# import AWS_Keys\n", "\n", "S3_CLIENT = boto3.client(\"s3\")\n", "STEPFUNCTION_CLIENT = boto3.client('stepfunctions')\n", @@ -202,16 +251,6 @@ "

1 - UPLOAD FIM4 HYDRO ID/FEATURE ID CROSSWALK

" ] }, - { - "cell_type": "markdown", - "id": "28195858-0cb6-4ad3-8966-db1371a8452a", - "metadata": { - "tags": [] - }, - "source": [ - "February 2024 Update from Tyler: This code will need to be updated to handle a new hand_id unique integer that the fim team (Rob Hanna and Matt Luck) has added to the crosswalk, and is now important to fim runs. They also changed the field names / format to match our schema, so this chunk of code should be able to be simplified significantly." - ] - }, { "cell_type": "code", "execution_count": null, @@ -221,12 +260,9 @@ }, "outputs": [], "source": [ - "\n", - " # This was already done for 4.4.0.0, so we can skip it jump from 4.5.2.0 to 4.2.5.11\n", - " \n", - "\n", + " \n", "'''\n", - "Be Very Careful to just rename tables. If they have indexes, the index will now point to the new\n", + "Be very careful to just rename tables. If they have indexes, the index will now point to the new\n", "table names but maintain the original index name. Those index names can really mess stuff up.\n", "Best to never rename unless you rename indexes as well. This particular on is ok. \n", "Note: When various '\"to_sql\" tools are run which have GIST indexes, this index column name issue\n", From c95e6c241d52ae25c6d2f9bd36a448b20b11d70e Mon Sep 17 00:00:00 2001 From: RobH Date: Thu, 31 Oct 2024 20:44:47 +0000 Subject: [PATCH 05/18] put back change on lambda function --- .../viz_db_ingest/lambda_function.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/Core/LAMBDA/viz_functions/viz_db_ingest/lambda_function.py b/Core/LAMBDA/viz_functions/viz_db_ingest/lambda_function.py index b0c29042..2941fa3a 100644 --- a/Core/LAMBDA/viz_functions/viz_db_ingest/lambda_function.py +++ b/Core/LAMBDA/viz_functions/viz_db_ingest/lambda_function.py @@ -62,26 +62,6 @@ def lambda_handler(event, context): ds = xr.open_dataset(download_path) ds_vars = [var for var in ds.variables] - match = re.search(r'common/data/model/com/nwm/prod/nwm.\d{8}/(.*)', file) - test_bucket = 'hydrovis-ti-deployment-us-east-1' - s3_fname = f'test_nwm_outputs/{match.group(1)}' - local_fname = f'/tmp/{match.group(1)}' - if 'forcing' in file: - interest_val = 'RAINRATE' - elif 'channel_rt' in file: - interest_val = 'streamflow' - elif 'coastal' in file: - interest_val = 'elevation' - elif 'reservoir' in file: - interest_val = 'water_sfc_elev' - - try: - check_if_file_exists(test_bucket, s3_fname, download=False) - except: - ds['streamflow'] = (ds['streamflow'] + 1) * 1000 - ds.to_netcdf(local_fname) - s3.upload_file(local_fname, test_bucket, s3_fname) - if not target_cols: target_cols = ds_vars From 4068da706b2db10cfc9d63046a40cc13601e764e Mon Sep 17 00:00:00 2001 From: RobH Date: Thu, 31 Oct 2024 21:23:31 +0000 Subject: [PATCH 06/18] WIP in changes to load script --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index ae79b335..0eb88a36 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "f15f66a5-1f40-4933-8601-b56e2e8134a9", + "id": "68fce701-8ba2-400d-b662-dca6439ef9b0", "metadata": {}, "source": [ "### Notes - Oct 21, 2024 ###\n", @@ -18,12 +18,12 @@ }, { "cell_type": "markdown", - "id": "d2741e09-a2f3-409c-adc1-bf6f28e18646", + "id": "ec59882d-62b5-4f2d-b9b5-4eb13df01f1e", "metadata": { "tags": [] }, "source": [ - "### Load Status for hand 4.5.11.1 - Started Oct 21, 2024\n", + "### Load Status for hand 4.5.11.1 - Started Oct 31, 2024 (well... restarted from teh 21st)\n", "\n", "#### Add dates to each line as they have been loaded\n", "\n", From dadb6007e38c4d0e67bcd25b627016d9faa0cc66 Mon Sep 17 00:00:00 2001 From: RobH Date: Thu, 7 Nov 2024 20:18:03 +0000 Subject: [PATCH 07/18] WIP: working on data load --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 2591 +++++++++++++++-- Core/Manual_Workflows/__init__.py | 0 Core/__init__.py | 0 3 files changed, 2291 insertions(+), 300 deletions(-) create mode 100644 Core/Manual_Workflows/__init__.py create mode 100644 Core/__init__.py diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 0eb88a36..26251c4b 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -18,20 +18,20 @@ }, { "cell_type": "markdown", - "id": "ec59882d-62b5-4f2d-b9b5-4eb13df01f1e", + "id": "ed4c0734-01e8-4f37-a718-402d9be7a076", "metadata": { "tags": [] }, "source": [ - "### Load Status for hand 4.5.11.1 - Started Oct 31, 2024 (well... restarted from teh 21st)\n", + "### Load Status for hand 4.5.11.1 - Started Oct 31, 2024 (well... restarted from the 21st)\n", "\n", "#### Add dates to each line as they have been loaded\n", "\n", - "1. `Crosswalk` : \n", - "2. `Lambda FIM_PREFIX` : \n", - "3. `Lambda FIM_VERSION and Memory` : \n", - "4. `ras2fim` : No update in this release. Keep the last load.\n", - "5. `AEP`\n", + "1. `Crosswalk` : --- Done: Nov 5\n", + "2. `Lambda FIM_PREFIX` : --- Done: Nov 5\n", + "3. `Lambda FIM_VERSION and Memory` : --- Done: Nov 5\n", + "4. `ras2fim` : No update in this release. But a few adjustments for fim_version and model_version here : -- Done: Nov 5\n", + "5. `AEP` --- (HOLD) - needs lambda and image updates (via tf)\n", " - `2 year` : -- \n", " - `5 year` : -- \n", " - `10 year` : -- \n", @@ -39,10 +39,10 @@ " - `50 year` : -- \n", " - `HW / High Water` : -- \n", " - `Change the hv-vpp-ti-viz-fim-data-prep Lambda memory back to 2048mb` : -- \n", - "6. `Catchments`\n", + "6. `Catchments` --- (HOLD) - needs lambda and image updates (via tf)\n", " - `Branch 0` : -- \n", " - `GMS` : -- \n", - "7. `usgs_elev_table` : -- \n", + "7. `usgs_elev_table` : -- Done Nov 7\n", "8. `hydrotable / hydrotable_staggered` : -- \n", "9. `usgs_rating_curve / usgs_rating_curves staggered` : -- \n", "10. `Skills Metrics` : -- \n", @@ -58,9 +58,12 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 9, "id": "9e1ee2b5-b109-49e8-8255-f06449b44ee6", "metadata": { + "jupyter": { + "source_hidden": true + }, "scrolled": true, "tags": [] }, @@ -69,28 +72,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: geopandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.0.1)\n", - "Requirement already satisfied: numpy>=1.22 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.1.2)\n", - "Requirement already satisfied: pyogrio>=0.7.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (0.10.0)\n", - "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (24.1)\n", - "Requirement already satisfied: pandas>=1.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.2.2)\n", - "Requirement already satisfied: pyproj>=3.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (3.7.0)\n", - "Requirement already satisfied: shapely>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.0.6)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2.9.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.1)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.1)\n", - "Requirement already satisfied: certifi in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pyogrio>=0.7.2->geopandas) (2024.7.4)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=1.4.0->geopandas) (1.16.0)\n", - "Collecting geoalchemy2\n", - " Downloading GeoAlchemy2-0.15.2-py3-none-any.whl.metadata (2.1 kB)\n", - "Requirement already satisfied: SQLAlchemy>=1.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (2.0.31)\n", - "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (24.1)\n", - "Requirement already satisfied: typing-extensions>=4.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (4.12.2)\n", - "Requirement already satisfied: greenlet!=0.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (3.0.3)\n", - "Downloading GeoAlchemy2-0.15.2-py3-none-any.whl (73 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.6/73.6 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hInstalling collected packages: geoalchemy2\n", - "Successfully installed geoalchemy2-0.15.2\n", + "Collecting python-dotenv\n", + " Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n", + "Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", + "Installing collected packages: python-dotenv\n", + "Successfully installed python-dotenv-1.0.1\n", "All loaded\n" ] } @@ -98,13 +84,15 @@ "source": [ "# Cell to manually pip reload a packages that the Jupyter engine not retained\n", "# !pip install numpy\n", - "!pip install geopandas\n", + "# !pip install geopandas\n", "# !pip install pyarrow\n", "# !pip install xarray\n", - "!pip install geoalchemy2\n", + "# !pip install geoalchemy2\n", "# !pip install contextily\n", "# !pip install rioxarray\n", - "print(\"All loaded\")" + "\n", + "!pip install python-dotenv\n", + "print(\"All loaded\")\n" ] }, { @@ -121,9 +109,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "id": "5b656259", "metadata": { + "jupyter": { + "source_hidden": true + }, "scrolled": true, "tags": [] }, @@ -152,9 +143,11 @@ "import os\n", "import codecs\n", "import csv\n", + "\n", "import sys\n", "\n", "from datetime import datetime\n", + "from pathlib import Path\n", "\n", "import boto3\n", "import geopandas as gpd\n", @@ -172,23 +165,46 @@ "from sqlalchemy.exc import DataError # yes, reduntant, fix it later\n", "from sqlalchemy.types import Text # yes, reduntant, fix it later\n", "\n", - "sys.path.append(os.path.join(os.path.abspath(''), '..'))\n", "\n", - "# import helper_functions.s3_shared_functions as s3_sf\n", - "# import helper_functions.shared_functions as sf\n", - "# import helper_functions.viz_classes\n", + "sys.path.append(os.path.join(os.path.abspath(''), '..'))\n", "\n", - "import s3_shared_functions as s3_sf\n", - "import shared_functions as sf\n", + "import helper_functions.shared_functions as sf\n", + "import helper_functions.s3_shared_functions as s3_sf\n", "\n", "from helper_functions.viz_classes import database\n", - "from helper_functions.viz_db_ingest import lambda_handler as execute_db_ingest\n", "\n", + "print(\"imports loaded\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fae5ca0f-e582-4ff2-89b6-087285433a97", + "metadata": { + "jupyter": { + "source_hidden": true + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Load AWS Keys\n", + "from dotenv import load_dotenv\n", + "aws_keys_path = os.path.join(Path.home(),\"SageMaker\", \"AWS_keys.env\")\n", + "print(f\"aws_keys are at {aws_keys_path}\")\n", + "load_dotenv(aws_keys_path)\n", + "\n", + "TI_ACCESS_KEY = os.environ['WF_TI_ACCESS_KEY']\n", + "TI_SECRET_KEY = os.environ['WF_TI_SECRET_KEY']\n", + "TI_TOKEN = os.environ['WF_TI_TOKEN']\n", + "\n", + "# I updated the file but it is not being honored in the enviro values\n", "\n", - "print(\"imports loaded\")\n", + "# print(TI_ACCESS_KEY)\n", + "# print(TI_SECRET_KEY)\n", + "# print(TI_TOKEN)\n", "\n", - "# Note: Aug 2024: Sometimes if you need to do the pip install above, you need to reload this twice.. must be a circular dependency ?? or forced pkg reload\n", - "# Not sure why yet" + "print(\"aws_keys loaded\")" ] }, { @@ -209,10 +225,9 @@ ], "source": [ "\n", - "OLD_MODEL_VERSION = \"HAND 4.5.2.11\"\n", - "OLD_PUBLIC_VERIONS = \"HAND 4.5.2.11\"\n", "PUBLIC_FIM_VERSION = \"FIM 5.1.0\"\n", "FIM_MODEL_VERSION = \"HAND 4.5.11.1\"\n", + "RAS2FIM_MODEL_VERSION = \"ras2fim 2.0.1.0\"\n", "\n", "FIM_ROOT_DPATH = \"fim/hand_4_5_11_1\"\n", "HAND_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/hand_datasets\"\n", @@ -228,11 +243,8 @@ "# Sometimes these credential values get updated. To find the latest correct values, go to your AWS Console log page and click on the \"Access Key\"\n", "# link to get the latest valid set. Using the \"AWS environment variables\" values.\n", "# If this is not set correctly, you will get an HTTP error 400 when you call S3 lower.\n", - "# You might also see an error of 'An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist.\" the creds are not correct\"\n", - "\n", - "# # Helps us get to the keys. Note: This was added Oct 16, 2024 and is untested\n", - "# sys.path.append(os.path.join(os.path.abspath(''), '../../../../AWS_Secret_keys'))\n", - "# import AWS_Keys\n", + "# You might also see an error of 'An error occurred (NoSuchKey) when calling the GetObject operation:\n", + "# The specified key does not exist.\" the creds are not correct\"\n", "\n", "S3_CLIENT = boto3.client(\"s3\")\n", "STEPFUNCTION_CLIENT = boto3.client('stepfunctions')\n", @@ -251,47 +263,14 @@ "

1 - UPLOAD FIM4 HYDRO ID/FEATURE ID CROSSWALK

" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "007c59f4-72ff-4e3e-9f99-36a8e830c426", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - " \n", - "'''\n", - "Be very careful to just rename tables. If they have indexes, the index will now point to the new\n", - "table names but maintain the original index name. Those index names can really mess stuff up.\n", - "Best to never rename unless you rename indexes as well. This particular on is ok. \n", - "Note: When various '\"to_sql\" tools are run which have GIST indexes, this index column name issue\n", - "will be the problem.\n", - "\n", - "Why Drop instead of Truncate? if the schema changes for the incoming, truncate will have column\n", - "missmatches.\n", - "\n", - "We really should be backing up indexes and constraints as well.\n", - "\n", - "'''\n", - "\n", - "\n", - "# TODO: Aug 2024: Change this to a backup without indexes and not rename, it affects indexes that we might need\n", - "# tbl_name = \"derived.fim4_featureid_crosswalk\"\n", - "# new_table_name = f\"{tbl_name}_{OLD_FIM_TAG}\"\n", - "# sql = f'''\n", - "# CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE {tbl_name};\n", - "# '''\n", - "# sf.execute_sql(sql)\n", - "# print(f\"{tbl_name} copied to {new_table_name} if it does not already exists\")\n", - "\n" - ] - }, { "cell_type": "code", "execution_count": null, "id": "c96a49f2", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -303,7 +282,7 @@ "d_reader = csv.DictReader(codecs.getreader(\"utf-8\")(data[\"Body\"]))\n", "headers = d_reader.fieldnames\n", "\n", - "# Aug 5, 2024 - Updated column names for 4.5.2.11\n", + "\n", "header_str = \"(\"\n", "for header in headers:\n", " header_str += header\n", @@ -323,6 +302,9 @@ " sql = f\"DROP TABLE IF EXISTS derived.fim4_featureid_crosswalk; CREATE TABLE derived.fim4_featureid_crosswalk {header_str};\"\n", " cur.execute(sql)\n", " conn.commit()\n", + " \n", + " # TODO: Nov: Drop the other 2 tables? No. ignore featureid_huc_crosswalk and featureid_huc_crosswalk_ak (not ours)\n", + " \n", "\n", " print(f\"Importing {FIM_CROSSWALK_FPATH} to derived.fim4_featureid_crosswalk\")\n", " sql = f\"\"\"\n", @@ -341,11 +323,8 @@ " \"\"\"\n", " cur.execute(sql)\n", " conn.commit()\n", - "\n", + " \n", " \n", - " # Aug 5, 2024: see notes at the top about the new FIM 5.0.0 system \n", - " # We will manually add a couple of new columns for public display\n", - " # New columns names are public_fim_version (FIM_5_0_0) and public model version (FIM_4_5_2_11)\n", " print(f\"Adding {COLUMN_NAME_FIM_VERSION} column to derived.fim4_featureid_crosswalk\")\n", " sql = f\"ALTER TABLE derived.fim4_featureid_crosswalk ADD COLUMN IF NOT EXISTS {COLUMN_NAME_FIM_VERSION} text DEFAULT '{PUBLIC_FIM_VERSION}';\"\n", " cur.execute(sql)\n", @@ -355,7 +334,7 @@ " sql = f\"ALTER TABLE derived.fim4_featureid_crosswalk ADD COLUMN IF NOT EXISTS {COLUMN_NAME_MODEL_VERSION} text DEFAULT '{FIM_MODEL_VERSION}';\"\n", " cur.execute(sql)\n", " conn.commit()\n", - "\n", + " \n", " print(\"Adding feature id index to derived.fim4_featureid_crosswalk\")\n", " # Drop it already exists\n", " sql = \"DROP INDEX IF EXISTS derived.fim4_crosswalk_feature_id\"\n", @@ -375,12 +354,13 @@ " conn.commit()\n", "\n", "print(\"\")\n", - "print(f\"Successully loaded derived.fim4_featureid_crosswalk and updated it\")\n" + "print(\"Successully loaded derived.fim4_featureid_crosswalk and updated it\")\n", + "print(\"... Estimated time to completion is just a few mins\")\n" ] }, { "cell_type": "markdown", - "id": "23500c40-b088-4b62-aa10-fe2f88c52ef9", + "id": "c17fdbfc-6f88-483d-a76d-29ebd525c5e5", "metadata": { "tags": [] }, @@ -393,16 +373,12 @@ "\n", "In the Configuration Tab, click on the `Environment variables` (left menu), then change the `FIX_PREFIX` to location of the latest hand_dataset you are working on. Referencial to S3 Bucket name.\n", "
\n", - "ie) fim/fim_4_5_2_11/hand_datasets\n", - "\n", - "Aug 5, 2024: changed my FIM_PREFIX:\n", - "
from: fim/fim_4_5_2_0/hand_datasets\n", - "
to: fim/fim_4_5_2_11/hand_datasets\n" + "ie) fim/fim_4_5_11_1/hand_datasets\n" ] }, { "cell_type": "markdown", - "id": "040cf1f3-cdfd-467a-9ac0-5478a285f032", + "id": "13507d85-ff04-42f3-bf72-b896d63d4679", "metadata": { "tags": [] }, @@ -415,11 +391,7 @@ "\n", "In the `Configuration` Tab, click on the `Environment variables` (left menu), then change the `FIM_VERSION` to the latest fim model version. \n", "
\n", - "ie) 4.5.2.11\n", - "

\n", - "Aug 5, 2024: changed my FIM_VERSION:\n", - "
from: 4.5.2.0\n", - "
to: 4.5.2.11\n", + "ie) 4.5.11.1\n", "

\n", "Then: Still in the Configuration Tab, now click on the `General Configuration` (left menu), followed \n", "by the `edit` button on the far right side, to get into the `General Configuration` page details.\n", @@ -427,14 +399,17 @@ "
Memory (text field) to 4096 (MB) and\n", "
Emphermeral Storage tp 1024 (MB)\n", "
\n", + "\n", "#### Note: Later in these steps we will change the Memory and Emphermal Storage back to default values, see below ####\n", + "Nov 5, 2024: Added new variable called \"MODEL_VERSION\" : \"HAND 4.5.11.1\". FIM_VERSION IS NOW: 5.1.0\n", "\n", - "Aug 5, 2024: changed my Memory (4096) and Emphermal (1024):\n" + "\n", + "\n" ] }, { "cell_type": "markdown", - "id": "840db921-0e13-4057-8e5a-1dd45f33d614", + "id": "9f19f7ac-61d1-4323-8138-6823da658aa3", "metadata": { "tags": [] }, @@ -442,8 +417,9 @@ "

4 - UPDATE RAS2FIM DATA (inc ras2fim boundaries) IN DB

\n", "\n", "As of Oct 2024, we have a new fim (hand) release covered in this file, but ras2fim does not have a new\n", - "release. ras2fim will likely be loaded as new datasets become available. The code for ras2fim is removed\n", - "here from the 4.5.2.11 set and will rebuilt as it''s own new separate load script when that happens.\n", + "release. ras2fim will likely be loaded as new datasets become available. \n", + "\n", + "***The code for ras2fim is removed here from the 4.5.2.11 set and will rebuilt as it''s own new separate load script when that happens.***\n", "\n", "However, we will have a few modifications for ras2fim data (not a reload) to help bring in the new\n", "fim_version and model_version columns. Those changes are included here.\n" @@ -451,38 +427,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "dfc0a44c-8b46-4db5-a9a0-c65e0bf1e20b", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Updated geocurves table to new FIM version value\n" + ] + } + ], "source": [ "\n", - "# Already done for 4.4.0.0 (4.5.2.11)\n", - "\n", - "\n", - "# TODO: Aug 2024: Change this to a backup without indexes and not rename\n", + "# Update \"geocurves\" to update the \"fim_version\" field to \"FIM 5.1.0:\n", "\n", + "print(f\"Updating geocurves table to fim_version of {PUBLIC_FIM_VERSION}\"\n", "\n", - "# By doing a backup, we are leaving the original tables with the indexes and we want to keep them with\n", - "# ras2fim as it loads geometry and without those pre-existing indexes, loading can fail\n", - "# tbl_name = \"ras2fim.geocurves\"\n", - "# new_table_name = f\"{tbl_name}_{OLD_FIM_TAG}\"\n", - "# sql = f\"CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE {tbl_name};\"\n", - "# sf.execute_sql(sql)\n", - "# print(f\"{tbl_name} copied to {new_table_name} if it does not already exists\")\n", + "sf.execute_sql(f'''\n", + "UPDATE\n", + " ras2fim.geocurves\n", + "SET\n", + " fim_version = '{PUBLIC_FIM_VERSION}';\n", + "''', db_type=\"viz\")\n", "\n", - "\n", - "# tbl_name = \"ras2fim.max_geocurves\"\n", - "# new_table_name = f\"{tbl_name}_{OLD_FIM_TAG}\"\n", - "# sql = f\"CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE {tbl_name};\"\n", - "# sf.execute_sql(sql)\n", - "# print(f\"{tbl_name} copied to {new_table_name} if it does not already exists\")\n", - "\n", - "\n", - "# TBD: Oct 21, 2024\n", - "# Do we need to do adjustments to the rasfim data for the new fim_version and model_version tables?\n" + "print(\"Updating done for geocurves\")\n", + "\n" ] }, { @@ -519,9 +495,6 @@ "execution_count": null, "id": "6a698067", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -583,9 +556,6 @@ "execution_count": null, "id": "f6d6ee69", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -611,9 +581,6 @@ "execution_count": null, "id": "a4f89d9a", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -639,9 +606,6 @@ "execution_count": null, "id": "791d1a8b", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -667,9 +631,6 @@ "execution_count": null, "id": "1bb87128", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -695,9 +656,6 @@ "execution_count": null, "id": "4832e4e0", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -723,9 +681,6 @@ "execution_count": null, "id": "187e83bc-ebbe-4615-a046-e0ef7b09ad3b", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -751,9 +706,6 @@ "cell_type": "markdown", "id": "2c810767-2f5d-46b2-860c-5d5c549f2e2a", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "source": [ @@ -787,9 +739,6 @@ "execution_count": null, "id": "c77d6e5b-f5fb-422f-a6c3-0f53743cd631", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -816,9 +765,6 @@ "execution_count": null, "id": "a7495759", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -941,9 +887,6 @@ "execution_count": null, "id": "3ea2a5be-9936-4abe-832a-8582f3c3af64", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -966,9 +909,6 @@ "execution_count": null, "id": "9ae8917f", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1090,35 +1030,2180 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "4e4eb95e-e147-4789-a76b-d8bda33bc3cb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Already run for 4.4.0.0 (4.5.2.11)\n", - "\n", - "# TODO: Aug 2024: Change this to a backup without indexes and not rename\n", - "\n", - "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.usgs_elev_table RENAME TO usgs_elev_table_fim_{OLD_FIM_TAG};')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", "metadata": { + "collapsed": true, "jupyter": { - "source_hidden": true + "outputs_hidden": true }, "scrolled": true, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing 1 of 1000 on page 1\n", + "Processing 2 of 1000 on page 1\n", + "Processing 3 of 1000 on page 1\n", + "Processing 4 of 1000 on page 1\n", + "Processing 5 of 1000 on page 1\n", + "Processing 6 of 1000 on page 1\n", + "Processing 7 of 1000 on page 1\n", + "Processing 8 of 1000 on page 1\n", + "Processing 9 of 1000 on page 1\n", + "Processing 10 of 1000 on page 1\n", + "Processing 11 of 1000 on page 1\n", + "Processing 12 of 1000 on page 1\n", + "Processing 13 of 1000 on page 1\n", + "Processing 14 of 1000 on page 1\n", + "Processing 15 of 1000 on page 1\n", + "Processing 16 of 1000 on page 1\n", + "Processing 17 of 1000 on page 1\n", + "Processing 18 of 1000 on page 1\n", + "Processing 19 of 1000 on page 1\n", + "Processing 20 of 1000 on page 1\n", + "Processing 21 of 1000 on page 1\n", + "Processing 22 of 1000 on page 1\n", + "Processing 23 of 1000 on page 1\n", + "Processing 24 of 1000 on page 1\n", + "Processing 25 of 1000 on page 1\n", + "Processing 26 of 1000 on page 1\n", + "Processing 27 of 1000 on page 1\n", + "Processing 28 of 1000 on page 1\n", + "Processing 29 of 1000 on page 1\n", + "Processing 30 of 1000 on page 1\n", + "Processing 31 of 1000 on page 1\n", + "Processing 32 of 1000 on page 1\n", + "Processing 33 of 1000 on page 1\n", + "Processing 34 of 1000 on page 1\n", + "Processing 35 of 1000 on page 1\n", + "Processing 36 of 1000 on page 1\n", + "Processing 37 of 1000 on page 1\n", + "Processing 38 of 1000 on page 1\n", + "Processing 39 of 1000 on page 1\n", + "Processing 40 of 1000 on page 1\n", + "Processing 41 of 1000 on page 1\n", + "Processing 42 of 1000 on page 1\n", + "Processing 43 of 1000 on page 1\n", + "Processing 44 of 1000 on page 1\n", + "Processing 45 of 1000 on page 1\n", + "Processing 46 of 1000 on page 1\n", + "Processing 47 of 1000 on page 1\n", + "Processing 48 of 1000 on page 1\n", + "Processing 49 of 1000 on page 1\n", + "Processing 50 of 1000 on page 1\n", + "Processing 51 of 1000 on page 1\n", + "Processing 52 of 1000 on page 1\n", + "Processing 53 of 1000 on page 1\n", + "Processing 54 of 1000 on page 1\n", + "Processing 55 of 1000 on page 1\n", + "Processing 56 of 1000 on page 1\n", + "Processing 57 of 1000 on page 1\n", + "Processing 58 of 1000 on page 1\n", + "Processing 59 of 1000 on page 1\n", + "Processing 60 of 1000 on page 1\n", + "Processing 61 of 1000 on page 1\n", + "Processing 62 of 1000 on page 1\n", + "Processing 63 of 1000 on page 1\n", + "Processing 64 of 1000 on page 1\n", + "Processing 65 of 1000 on page 1\n", + "Processing 66 of 1000 on page 1\n", + "Processing 67 of 1000 on page 1\n", + "Processing 68 of 1000 on page 1\n", + "Processing 69 of 1000 on page 1\n", + "Processing 70 of 1000 on page 1\n", + "Processing 71 of 1000 on page 1\n", + "Processing 72 of 1000 on page 1\n", + "Processing 73 of 1000 on page 1\n", + "Processing 74 of 1000 on page 1\n", + "Processing 75 of 1000 on page 1\n", + "Processing 76 of 1000 on page 1\n", + "Processing 77 of 1000 on page 1\n", + "Processing 78 of 1000 on page 1\n", + "Processing 79 of 1000 on page 1\n", + "Processing 80 of 1000 on page 1\n", + "Processing 81 of 1000 on page 1\n", + "Processing 82 of 1000 on page 1\n", + "Processing 83 of 1000 on page 1\n", + "Processing 84 of 1000 on page 1\n", + "Processing 85 of 1000 on page 1\n", + "Processing 86 of 1000 on page 1\n", + "Processing 87 of 1000 on page 1\n", + "Processing 88 of 1000 on page 1\n", + "Processing 89 of 1000 on page 1\n", + "Processing 90 of 1000 on page 1\n", + "Processing 91 of 1000 on page 1\n", + "Processing 92 of 1000 on page 1\n", + "Processing 93 of 1000 on page 1\n", + "Processing 94 of 1000 on page 1\n", + "Processing 95 of 1000 on page 1\n", + "Processing 96 of 1000 on page 1\n", + "Processing 97 of 1000 on page 1\n", + "Processing 98 of 1000 on page 1\n", + "Processing 99 of 1000 on page 1\n", + "Processing 100 of 1000 on page 1\n", + "Processing 101 of 1000 on page 1\n", + "Processing 102 of 1000 on page 1\n", + "Processing 103 of 1000 on page 1\n", + "Processing 104 of 1000 on page 1\n", + "Processing 105 of 1000 on page 1\n", + "Processing 106 of 1000 on page 1\n", + "Processing 107 of 1000 on page 1\n", + "Processing 108 of 1000 on page 1\n", + "Processing 109 of 1000 on page 1\n", + "Processing 110 of 1000 on page 1\n", + "Processing 111 of 1000 on page 1\n", + "Processing 112 of 1000 on page 1\n", + "Processing 113 of 1000 on page 1\n", + "Processing 114 of 1000 on page 1\n", + "Processing 115 of 1000 on page 1\n", + "Processing 116 of 1000 on page 1\n", + "Processing 117 of 1000 on page 1\n", + "Processing 118 of 1000 on page 1\n", + "Processing 119 of 1000 on page 1\n", + "Processing 120 of 1000 on page 1\n", + "Processing 121 of 1000 on page 1\n", + "Processing 122 of 1000 on page 1\n", + "Processing 123 of 1000 on page 1\n", + "Processing 124 of 1000 on page 1\n", + "Processing 125 of 1000 on page 1\n", + "Processing 126 of 1000 on page 1\n", + "Processing 127 of 1000 on page 1\n", + "Processing 128 of 1000 on page 1\n", + "Processing 129 of 1000 on page 1\n", + "Processing 130 of 1000 on page 1\n", + "Processing 131 of 1000 on page 1\n", + "Processing 132 of 1000 on page 1\n", + "Processing 133 of 1000 on page 1\n", + "Processing 134 of 1000 on page 1\n", + "Processing 135 of 1000 on page 1\n", + "Processing 136 of 1000 on page 1\n", + "Processing 137 of 1000 on page 1\n", + "Processing 138 of 1000 on page 1\n", + "Processing 139 of 1000 on page 1\n", + "Processing 140 of 1000 on page 1\n", + "Processing 141 of 1000 on page 1\n", + "Processing 142 of 1000 on page 1\n", + "Processing 143 of 1000 on page 1\n", + "Processing 144 of 1000 on page 1\n", + "Processing 145 of 1000 on page 1\n", + "Processing 146 of 1000 on page 1\n", + "Processing 147 of 1000 on page 1\n", + "Processing 148 of 1000 on page 1\n", + "Processing 149 of 1000 on page 1\n", + "Processing 150 of 1000 on page 1\n", + "Processing 151 of 1000 on page 1\n", + "Processing 152 of 1000 on page 1\n", + "Processing 153 of 1000 on page 1\n", + "Processing 154 of 1000 on page 1\n", + "Processing 155 of 1000 on page 1\n", + "Processing 156 of 1000 on page 1\n", + "Processing 157 of 1000 on page 1\n", + "Processing 158 of 1000 on page 1\n", + "Processing 159 of 1000 on page 1\n", + "Processing 160 of 1000 on page 1\n", + "Processing 161 of 1000 on page 1\n", + "Processing 162 of 1000 on page 1\n", + "Processing 163 of 1000 on page 1\n", + "Processing 164 of 1000 on page 1\n", + "Processing 165 of 1000 on page 1\n", + "Processing 166 of 1000 on page 1\n", + "Processing 167 of 1000 on page 1\n", + "Processing 168 of 1000 on page 1\n", + "Processing 169 of 1000 on page 1\n", + "Processing 170 of 1000 on page 1\n", + "Processing 171 of 1000 on page 1\n", + "Processing 172 of 1000 on page 1\n", + "Processing 173 of 1000 on page 1\n", + "Processing 174 of 1000 on page 1\n", + "Processing 175 of 1000 on page 1\n", + "Processing 176 of 1000 on page 1\n", + "Processing 177 of 1000 on page 1\n", + "Processing 178 of 1000 on page 1\n", + "Processing 179 of 1000 on page 1\n", + "Processing 180 of 1000 on page 1\n", + "Processing 181 of 1000 on page 1\n", + "Processing 182 of 1000 on page 1\n", + "Processing 183 of 1000 on page 1\n", + "Processing 184 of 1000 on page 1\n", + "Processing 185 of 1000 on page 1\n", + "Processing 186 of 1000 on page 1\n", + "Processing 187 of 1000 on page 1\n", + "Processing 188 of 1000 on page 1\n", + "Processing 189 of 1000 on page 1\n", + "Processing 190 of 1000 on page 1\n", + "Processing 191 of 1000 on page 1\n", + "Processing 192 of 1000 on page 1\n", + "Processing 193 of 1000 on page 1\n", + "Processing 194 of 1000 on page 1\n", + "Processing 195 of 1000 on page 1\n", + "Processing 196 of 1000 on page 1\n", + "Processing 197 of 1000 on page 1\n", + "Processing 198 of 1000 on page 1\n", + "Processing 199 of 1000 on page 1\n", + "Processing 200 of 1000 on page 1\n", + "Processing 201 of 1000 on page 1\n", + "Processing 202 of 1000 on page 1\n", + "Processing 203 of 1000 on page 1\n", + "Processing 204 of 1000 on page 1\n", + "Processing 205 of 1000 on page 1\n", + "Processing 206 of 1000 on page 1\n", + "Processing 207 of 1000 on page 1\n", + "Processing 208 of 1000 on page 1\n", + "Processing 209 of 1000 on page 1\n", + "Processing 210 of 1000 on page 1\n", + "Processing 211 of 1000 on page 1\n", + "Processing 212 of 1000 on page 1\n", + "Processing 213 of 1000 on page 1\n", + "Processing 214 of 1000 on page 1\n", + "Processing 215 of 1000 on page 1\n", + "Processing 216 of 1000 on page 1\n", + "Processing 217 of 1000 on page 1\n", + "Processing 218 of 1000 on page 1\n", + "Processing 219 of 1000 on page 1\n", + "Processing 220 of 1000 on page 1\n", + "Processing 221 of 1000 on page 1\n", + "Processing 222 of 1000 on page 1\n", + "Processing 223 of 1000 on page 1\n", + "Processing 224 of 1000 on page 1\n", + "Processing 225 of 1000 on page 1\n", + "Processing 226 of 1000 on page 1\n", + "Processing 227 of 1000 on page 1\n", + "Processing 228 of 1000 on page 1\n", + "Processing 229 of 1000 on page 1\n", + "Processing 230 of 1000 on page 1\n", + "Processing 231 of 1000 on page 1\n", + "Processing 232 of 1000 on page 1\n", + "Processing 233 of 1000 on page 1\n", + "Processing 234 of 1000 on page 1\n", + "Processing 235 of 1000 on page 1\n", + "Processing 236 of 1000 on page 1\n", + "Processing 237 of 1000 on page 1\n", + "Processing 238 of 1000 on page 1\n", + "Processing 239 of 1000 on page 1\n", + "Processing 240 of 1000 on page 1\n", + "Processing 241 of 1000 on page 1\n", + "Processing 242 of 1000 on page 1\n", + "Processing 243 of 1000 on page 1\n", + "Processing 244 of 1000 on page 1\n", + "Processing 245 of 1000 on page 1\n", + "Processing 246 of 1000 on page 1\n", + "Processing 247 of 1000 on page 1\n", + "Processing 248 of 1000 on page 1\n", + "Processing 249 of 1000 on page 1\n", + "Processing 250 of 1000 on page 1\n", + "Processing 251 of 1000 on page 1\n", + "Processing 252 of 1000 on page 1\n", + "Processing 253 of 1000 on page 1\n", + "Processing 254 of 1000 on page 1\n", + "Processing 255 of 1000 on page 1\n", + "Processing 256 of 1000 on page 1\n", + "Processing 257 of 1000 on page 1\n", + "Processing 258 of 1000 on page 1\n", + "Processing 259 of 1000 on page 1\n", + "Processing 260 of 1000 on page 1\n", + "Processing 261 of 1000 on page 1\n", + "Processing 262 of 1000 on page 1\n", + "Processing 263 of 1000 on page 1\n", + "Processing 264 of 1000 on page 1\n", + "Processing 265 of 1000 on page 1\n", + "Processing 266 of 1000 on page 1\n", + "Processing 267 of 1000 on page 1\n", + "Processing 268 of 1000 on page 1\n", + "Processing 269 of 1000 on page 1\n", + "Processing 270 of 1000 on page 1\n", + "Processing 271 of 1000 on page 1\n", + "Processing 272 of 1000 on page 1\n", + "Processing 273 of 1000 on page 1\n", + "Processing 274 of 1000 on page 1\n", + "Processing 275 of 1000 on page 1\n", + "Processing 276 of 1000 on page 1\n", + "Processing 277 of 1000 on page 1\n", + "Processing 278 of 1000 on page 1\n", + "Processing 279 of 1000 on page 1\n", + "Processing 280 of 1000 on page 1\n", + "Processing 281 of 1000 on page 1\n", + "Processing 282 of 1000 on page 1\n", + "Processing 283 of 1000 on page 1\n", + "Processing 284 of 1000 on page 1\n", + "Processing 285 of 1000 on page 1\n", + "Processing 286 of 1000 on page 1\n", + "Processing 287 of 1000 on page 1\n", + "Processing 288 of 1000 on page 1\n", + "Processing 289 of 1000 on page 1\n", + "Processing 290 of 1000 on page 1\n", + "Processing 291 of 1000 on page 1\n", + "Processing 292 of 1000 on page 1\n", + "Processing 293 of 1000 on page 1\n", + "Processing 294 of 1000 on page 1\n", + "Processing 295 of 1000 on page 1\n", + "Processing 296 of 1000 on page 1\n", + "Processing 297 of 1000 on page 1\n", + "Processing 298 of 1000 on page 1\n", + "Processing 299 of 1000 on page 1\n", + "Processing 300 of 1000 on page 1\n", + "Processing 301 of 1000 on page 1\n", + "Processing 302 of 1000 on page 1\n", + "Processing 303 of 1000 on page 1\n", + "Processing 304 of 1000 on page 1\n", + "Processing 305 of 1000 on page 1\n", + "Processing 306 of 1000 on page 1\n", + "Processing 307 of 1000 on page 1\n", + "Processing 308 of 1000 on page 1\n", + "Processing 309 of 1000 on page 1\n", + "Processing 310 of 1000 on page 1\n", + "Processing 311 of 1000 on page 1\n", + "Processing 312 of 1000 on page 1\n", + "Processing 313 of 1000 on page 1\n", + "Processing 314 of 1000 on page 1\n", + "Processing 315 of 1000 on page 1\n", + "Processing 316 of 1000 on page 1\n", + "Processing 317 of 1000 on page 1\n", + "Processing 318 of 1000 on page 1\n", + "Processing 319 of 1000 on page 1\n", + "Processing 320 of 1000 on page 1\n", + "Processing 321 of 1000 on page 1\n", + "Processing 322 of 1000 on page 1\n", + "Processing 323 of 1000 on page 1\n", + "Processing 324 of 1000 on page 1\n", + "Processing 325 of 1000 on page 1\n", + "Processing 326 of 1000 on page 1\n", + "Processing 327 of 1000 on page 1\n", + "Processing 328 of 1000 on page 1\n", + "Processing 329 of 1000 on page 1\n", + "Processing 330 of 1000 on page 1\n", + "Processing 331 of 1000 on page 1\n", + "Processing 332 of 1000 on page 1\n", + "Processing 333 of 1000 on page 1\n", + "Processing 334 of 1000 on page 1\n", + "Processing 335 of 1000 on page 1\n", + "Processing 336 of 1000 on page 1\n", + "Processing 337 of 1000 on page 1\n", + "Processing 338 of 1000 on page 1\n", + "Processing 339 of 1000 on page 1\n", + "Processing 340 of 1000 on page 1\n", + "Processing 341 of 1000 on page 1\n", + "Processing 342 of 1000 on page 1\n", + "Processing 343 of 1000 on page 1\n", + "Processing 344 of 1000 on page 1\n", + "Processing 345 of 1000 on page 1\n", + "Processing 346 of 1000 on page 1\n", + "Processing 347 of 1000 on page 1\n", + "Processing 348 of 1000 on page 1\n", + "Processing 349 of 1000 on page 1\n", + "Processing 350 of 1000 on page 1\n", + "Processing 351 of 1000 on page 1\n", + "Processing 352 of 1000 on page 1\n", + "Processing 353 of 1000 on page 1\n", + "Processing 354 of 1000 on page 1\n", + "Processing 355 of 1000 on page 1\n", + "Processing 356 of 1000 on page 1\n", + "Processing 357 of 1000 on page 1\n", + "Processing 358 of 1000 on page 1\n", + "Processing 359 of 1000 on page 1\n", + "Processing 360 of 1000 on page 1\n", + "Processing 361 of 1000 on page 1\n", + "Processing 362 of 1000 on page 1\n", + "Processing 363 of 1000 on page 1\n", + "Processing 364 of 1000 on page 1\n", + "Processing 365 of 1000 on page 1\n", + "Processing 366 of 1000 on page 1\n", + "Processing 367 of 1000 on page 1\n", + "Processing 368 of 1000 on page 1\n", + "Processing 369 of 1000 on page 1\n", + "Processing 370 of 1000 on page 1\n", + "Processing 371 of 1000 on page 1\n", + "Processing 372 of 1000 on page 1\n", + "Processing 373 of 1000 on page 1\n", + "Processing 374 of 1000 on page 1\n", + "Processing 375 of 1000 on page 1\n", + "Processing 376 of 1000 on page 1\n", + "Processing 377 of 1000 on page 1\n", + "Processing 378 of 1000 on page 1\n", + "Processing 379 of 1000 on page 1\n", + "Processing 380 of 1000 on page 1\n", + "Processing 381 of 1000 on page 1\n", + "Processing 382 of 1000 on page 1\n", + "Processing 383 of 1000 on page 1\n", + "Processing 384 of 1000 on page 1\n", + "Processing 385 of 1000 on page 1\n", + "Processing 386 of 1000 on page 1\n", + "Processing 387 of 1000 on page 1\n", + "Processing 388 of 1000 on page 1\n", + "Processing 389 of 1000 on page 1\n", + "Processing 390 of 1000 on page 1\n", + "Processing 391 of 1000 on page 1\n", + "Processing 392 of 1000 on page 1\n", + "Processing 393 of 1000 on page 1\n", + "Processing 394 of 1000 on page 1\n", + "Processing 395 of 1000 on page 1\n", + "Processing 396 of 1000 on page 1\n", + "Processing 397 of 1000 on page 1\n", + "Processing 398 of 1000 on page 1\n", + "Processing 399 of 1000 on page 1\n", + "Processing 400 of 1000 on page 1\n", + "Processing 401 of 1000 on page 1\n", + "Processing 402 of 1000 on page 1\n", + "Processing 403 of 1000 on page 1\n", + "Processing 404 of 1000 on page 1\n", + "Processing 405 of 1000 on page 1\n", + "Processing 406 of 1000 on page 1\n", + "Processing 407 of 1000 on page 1\n", + "Processing 408 of 1000 on page 1\n", + "Processing 409 of 1000 on page 1\n", + "Processing 410 of 1000 on page 1\n", + "Processing 411 of 1000 on page 1\n", + "Processing 412 of 1000 on page 1\n", + "Processing 413 of 1000 on page 1\n", + "Processing 414 of 1000 on page 1\n", + "Processing 415 of 1000 on page 1\n", + "Processing 416 of 1000 on page 1\n", + "Processing 417 of 1000 on page 1\n", + "Processing 418 of 1000 on page 1\n", + "Processing 419 of 1000 on page 1\n", + "Processing 420 of 1000 on page 1\n", + "Processing 421 of 1000 on page 1\n", + "Processing 422 of 1000 on page 1\n", + "Processing 423 of 1000 on page 1\n", + "Processing 424 of 1000 on page 1\n", + "Processing 425 of 1000 on page 1\n", + "Processing 426 of 1000 on page 1\n", + "Processing 427 of 1000 on page 1\n", + "Processing 428 of 1000 on page 1\n", + "Processing 429 of 1000 on page 1\n", + "Processing 430 of 1000 on page 1\n", + "Processing 431 of 1000 on page 1\n", + "Processing 432 of 1000 on page 1\n", + "Processing 433 of 1000 on page 1\n", + "Processing 434 of 1000 on page 1\n", + "Processing 435 of 1000 on page 1\n", + "Processing 436 of 1000 on page 1\n", + "Processing 437 of 1000 on page 1\n", + "Processing 438 of 1000 on page 1\n", + "Processing 439 of 1000 on page 1\n", + "Processing 440 of 1000 on page 1\n", + "Processing 441 of 1000 on page 1\n", + "Processing 442 of 1000 on page 1\n", + "Processing 443 of 1000 on page 1\n", + "Processing 444 of 1000 on page 1\n", + "Processing 445 of 1000 on page 1\n", + "Processing 446 of 1000 on page 1\n", + "Processing 447 of 1000 on page 1\n", + "Processing 448 of 1000 on page 1\n", + "Processing 449 of 1000 on page 1\n", + "Processing 450 of 1000 on page 1\n", + "Processing 451 of 1000 on page 1\n", + "Processing 452 of 1000 on page 1\n", + "Processing 453 of 1000 on page 1\n", + "Processing 454 of 1000 on page 1\n", + "Processing 455 of 1000 on page 1\n", + "Processing 456 of 1000 on page 1\n", + "Processing 457 of 1000 on page 1\n", + "Processing 458 of 1000 on page 1\n", + "Processing 459 of 1000 on page 1\n", + "Processing 460 of 1000 on page 1\n", + "Processing 461 of 1000 on page 1\n", + "Processing 462 of 1000 on page 1\n", + "Processing 463 of 1000 on page 1\n", + "Processing 464 of 1000 on page 1\n", + "Processing 465 of 1000 on page 1\n", + "Processing 466 of 1000 on page 1\n", + "Processing 467 of 1000 on page 1\n", + "Processing 468 of 1000 on page 1\n", + "Processing 469 of 1000 on page 1\n", + "Processing 470 of 1000 on page 1\n", + "Processing 471 of 1000 on page 1\n", + "Processing 472 of 1000 on page 1\n", + "Processing 473 of 1000 on page 1\n", + "Processing 474 of 1000 on page 1\n", + "Processing 475 of 1000 on page 1\n", + "Processing 476 of 1000 on page 1\n", + "Processing 477 of 1000 on page 1\n", + "Processing 478 of 1000 on page 1\n", + "Processing 479 of 1000 on page 1\n", + "Processing 480 of 1000 on page 1\n", + "Processing 481 of 1000 on page 1\n", + "Processing 482 of 1000 on page 1\n", + "Processing 483 of 1000 on page 1\n", + "Processing 484 of 1000 on page 1\n", + "Processing 485 of 1000 on page 1\n", + "Processing 486 of 1000 on page 1\n", + "Processing 487 of 1000 on page 1\n", + "Processing 488 of 1000 on page 1\n", + "Processing 489 of 1000 on page 1\n", + "Processing 490 of 1000 on page 1\n", + "Processing 491 of 1000 on page 1\n", + "Processing 492 of 1000 on page 1\n", + "Processing 493 of 1000 on page 1\n", + "Processing 494 of 1000 on page 1\n", + "Processing 495 of 1000 on page 1\n", + "Processing 496 of 1000 on page 1\n", + "Processing 497 of 1000 on page 1\n", + "Processing 498 of 1000 on page 1\n", + "Processing 499 of 1000 on page 1\n", + "Processing 500 of 1000 on page 1\n", + "Processing 501 of 1000 on page 1\n", + "Processing 502 of 1000 on page 1\n", + "Processing 503 of 1000 on page 1\n", + "Processing 504 of 1000 on page 1\n", + "Processing 505 of 1000 on page 1\n", + "Processing 506 of 1000 on page 1\n", + "Processing 507 of 1000 on page 1\n", + "Processing 508 of 1000 on page 1\n", + "Processing 509 of 1000 on page 1\n", + "Processing 510 of 1000 on page 1\n", + "Processing 511 of 1000 on page 1\n", + "Processing 512 of 1000 on page 1\n", + "Processing 513 of 1000 on page 1\n", + "Processing 514 of 1000 on page 1\n", + "Processing 515 of 1000 on page 1\n", + "Processing 516 of 1000 on page 1\n", + "Processing 517 of 1000 on page 1\n", + "Processing 518 of 1000 on page 1\n", + "Processing 519 of 1000 on page 1\n", + "Processing 520 of 1000 on page 1\n", + "Processing 521 of 1000 on page 1\n", + "Processing 522 of 1000 on page 1\n", + "Processing 523 of 1000 on page 1\n", + "Processing 524 of 1000 on page 1\n", + "Processing 525 of 1000 on page 1\n", + "Processing 526 of 1000 on page 1\n", + "Processing 527 of 1000 on page 1\n", + "Processing 528 of 1000 on page 1\n", + "Processing 529 of 1000 on page 1\n", + "Processing 530 of 1000 on page 1\n", + "Processing 531 of 1000 on page 1\n", + "Processing 532 of 1000 on page 1\n", + "Processing 533 of 1000 on page 1\n", + "Processing 534 of 1000 on page 1\n", + "Processing 535 of 1000 on page 1\n", + "Processing 536 of 1000 on page 1\n", + "Processing 537 of 1000 on page 1\n", + "Processing 538 of 1000 on page 1\n", + "Processing 539 of 1000 on page 1\n", + "Processing 540 of 1000 on page 1\n", + "Processing 541 of 1000 on page 1\n", + "Processing 542 of 1000 on page 1\n", + "Processing 543 of 1000 on page 1\n", + "Processing 544 of 1000 on page 1\n", + "Processing 545 of 1000 on page 1\n", + "Processing 546 of 1000 on page 1\n", + "Processing 547 of 1000 on page 1\n", + "Processing 548 of 1000 on page 1\n", + "Processing 549 of 1000 on page 1\n", + "Processing 550 of 1000 on page 1\n", + "Processing 551 of 1000 on page 1\n", + "Processing 552 of 1000 on page 1\n", + "Processing 553 of 1000 on page 1\n", + "Processing 554 of 1000 on page 1\n", + "Processing 555 of 1000 on page 1\n", + "Processing 556 of 1000 on page 1\n", + "Processing 557 of 1000 on page 1\n", + "Processing 558 of 1000 on page 1\n", + "Processing 559 of 1000 on page 1\n", + "Processing 560 of 1000 on page 1\n", + "Processing 561 of 1000 on page 1\n", + "Processing 562 of 1000 on page 1\n", + "Processing 563 of 1000 on page 1\n", + "Processing 564 of 1000 on page 1\n", + "Processing 565 of 1000 on page 1\n", + "Processing 566 of 1000 on page 1\n", + "Processing 567 of 1000 on page 1\n", + "Processing 568 of 1000 on page 1\n", + "Processing 569 of 1000 on page 1\n", + "Processing 570 of 1000 on page 1\n", + "Processing 571 of 1000 on page 1\n", + "Processing 572 of 1000 on page 1\n", + "Processing 573 of 1000 on page 1\n", + "Processing 574 of 1000 on page 1\n", + "Processing 575 of 1000 on page 1\n", + "Processing 576 of 1000 on page 1\n", + "Processing 577 of 1000 on page 1\n", + "Processing 578 of 1000 on page 1\n", + "Processing 579 of 1000 on page 1\n", + "Processing 580 of 1000 on page 1\n", + "Processing 581 of 1000 on page 1\n", + "Processing 582 of 1000 on page 1\n", + "Processing 583 of 1000 on page 1\n", + "Processing 584 of 1000 on page 1\n", + "Processing 585 of 1000 on page 1\n", + "Processing 586 of 1000 on page 1\n", + "Processing 587 of 1000 on page 1\n", + "Processing 588 of 1000 on page 1\n", + "Processing 589 of 1000 on page 1\n", + "Processing 590 of 1000 on page 1\n", + "Processing 591 of 1000 on page 1\n", + "Processing 592 of 1000 on page 1\n", + "Processing 593 of 1000 on page 1\n", + "Processing 594 of 1000 on page 1\n", + "Processing 595 of 1000 on page 1\n", + "Processing 596 of 1000 on page 1\n", + "Processing 597 of 1000 on page 1\n", + "Processing 598 of 1000 on page 1\n", + "Processing 599 of 1000 on page 1\n", + "Processing 600 of 1000 on page 1\n", + "Processing 601 of 1000 on page 1\n", + "Processing 602 of 1000 on page 1\n", + "Processing 603 of 1000 on page 1\n", + "Processing 604 of 1000 on page 1\n", + "Processing 605 of 1000 on page 1\n", + "Processing 606 of 1000 on page 1\n", + "Processing 607 of 1000 on page 1\n", + "Processing 608 of 1000 on page 1\n", + "Processing 609 of 1000 on page 1\n", + "Processing 610 of 1000 on page 1\n", + "Processing 611 of 1000 on page 1\n", + "Processing 612 of 1000 on page 1\n", + "Processing 613 of 1000 on page 1\n", + "Processing 614 of 1000 on page 1\n", + "Processing 615 of 1000 on page 1\n", + "Processing 616 of 1000 on page 1\n", + "Processing 617 of 1000 on page 1\n", + "Processing 618 of 1000 on page 1\n", + "Processing 619 of 1000 on page 1\n", + "Processing 620 of 1000 on page 1\n", + "Processing 621 of 1000 on page 1\n", + "Processing 622 of 1000 on page 1\n", + "Processing 623 of 1000 on page 1\n", + "Processing 624 of 1000 on page 1\n", + "Processing 625 of 1000 on page 1\n", + "Processing 626 of 1000 on page 1\n", + "Processing 627 of 1000 on page 1\n", + "Processing 628 of 1000 on page 1\n", + "Processing 629 of 1000 on page 1\n", + "Processing 630 of 1000 on page 1\n", + "Processing 631 of 1000 on page 1\n", + "Processing 632 of 1000 on page 1\n", + "Processing 633 of 1000 on page 1\n", + "Processing 634 of 1000 on page 1\n", + "Processing 635 of 1000 on page 1\n", + "Processing 636 of 1000 on page 1\n", + "Processing 637 of 1000 on page 1\n", + "Processing 638 of 1000 on page 1\n", + "Processing 639 of 1000 on page 1\n", + "Processing 640 of 1000 on page 1\n", + "Processing 641 of 1000 on page 1\n", + "Processing 642 of 1000 on page 1\n", + "Processing 643 of 1000 on page 1\n", + "Processing 644 of 1000 on page 1\n", + "Processing 645 of 1000 on page 1\n", + "Processing 646 of 1000 on page 1\n", + "Processing 647 of 1000 on page 1\n", + "Processing 648 of 1000 on page 1\n", + "Processing 649 of 1000 on page 1\n", + "Processing 650 of 1000 on page 1\n", + "Processing 651 of 1000 on page 1\n", + "Processing 652 of 1000 on page 1\n", + "Processing 653 of 1000 on page 1\n", + "Processing 654 of 1000 on page 1\n", + "Processing 655 of 1000 on page 1\n", + "Processing 656 of 1000 on page 1\n", + "Processing 657 of 1000 on page 1\n", + "Processing 658 of 1000 on page 1\n", + "Processing 659 of 1000 on page 1\n", + "Processing 660 of 1000 on page 1\n", + "Processing 661 of 1000 on page 1\n", + "Processing 662 of 1000 on page 1\n", + "Processing 663 of 1000 on page 1\n", + "Processing 664 of 1000 on page 1\n", + "Processing 665 of 1000 on page 1\n", + "Processing 666 of 1000 on page 1\n", + "Processing 667 of 1000 on page 1\n", + "Processing 668 of 1000 on page 1\n", + "Processing 669 of 1000 on page 1\n", + "Processing 670 of 1000 on page 1\n", + "Processing 671 of 1000 on page 1\n", + "Processing 672 of 1000 on page 1\n", + "Processing 673 of 1000 on page 1\n", + "Processing 674 of 1000 on page 1\n", + "Processing 675 of 1000 on page 1\n", + "Processing 676 of 1000 on page 1\n", + "Processing 677 of 1000 on page 1\n", + "Processing 678 of 1000 on page 1\n", + "Processing 679 of 1000 on page 1\n", + "Processing 680 of 1000 on page 1\n", + "Processing 681 of 1000 on page 1\n", + "Processing 682 of 1000 on page 1\n", + "Processing 683 of 1000 on page 1\n", + "Processing 684 of 1000 on page 1\n", + "Processing 685 of 1000 on page 1\n", + "Processing 686 of 1000 on page 1\n", + "Processing 687 of 1000 on page 1\n", + "Processing 688 of 1000 on page 1\n", + "Processing 689 of 1000 on page 1\n", + "Processing 690 of 1000 on page 1\n", + "Processing 691 of 1000 on page 1\n", + "Processing 692 of 1000 on page 1\n", + "Processing 693 of 1000 on page 1\n", + "Processing 694 of 1000 on page 1\n", + "Processing 695 of 1000 on page 1\n", + "Processing 696 of 1000 on page 1\n", + "Processing 697 of 1000 on page 1\n", + "Processing 698 of 1000 on page 1\n", + "Processing 699 of 1000 on page 1\n", + "Processing 700 of 1000 on page 1\n", + "Processing 701 of 1000 on page 1\n", + "Processing 702 of 1000 on page 1\n", + "Processing 703 of 1000 on page 1\n", + "Processing 704 of 1000 on page 1\n", + "Processing 705 of 1000 on page 1\n", + "Processing 706 of 1000 on page 1\n", + "Processing 707 of 1000 on page 1\n", + "Processing 708 of 1000 on page 1\n", + "Processing 709 of 1000 on page 1\n", + "Processing 710 of 1000 on page 1\n", + "Processing 711 of 1000 on page 1\n", + "Processing 712 of 1000 on page 1\n", + "Processing 713 of 1000 on page 1\n", + "Processing 714 of 1000 on page 1\n", + "Processing 715 of 1000 on page 1\n", + "Processing 716 of 1000 on page 1\n", + "Processing 717 of 1000 on page 1\n", + "Processing 718 of 1000 on page 1\n", + "Processing 719 of 1000 on page 1\n", + "Processing 720 of 1000 on page 1\n", + "Processing 721 of 1000 on page 1\n", + "Processing 722 of 1000 on page 1\n", + "Processing 723 of 1000 on page 1\n", + "Processing 724 of 1000 on page 1\n", + "Processing 725 of 1000 on page 1\n", + "Processing 726 of 1000 on page 1\n", + "Processing 727 of 1000 on page 1\n", + "Processing 728 of 1000 on page 1\n", + "Processing 729 of 1000 on page 1\n", + "Processing 730 of 1000 on page 1\n", + "Processing 731 of 1000 on page 1\n", + "Processing 732 of 1000 on page 1\n", + "Processing 733 of 1000 on page 1\n", + "Processing 734 of 1000 on page 1\n", + "Processing 735 of 1000 on page 1\n", + "Processing 736 of 1000 on page 1\n", + "Processing 737 of 1000 on page 1\n", + "Processing 738 of 1000 on page 1\n", + "Processing 739 of 1000 on page 1\n", + "Processing 740 of 1000 on page 1\n", + "Processing 741 of 1000 on page 1\n", + "Processing 742 of 1000 on page 1\n", + "Processing 743 of 1000 on page 1\n", + "Processing 744 of 1000 on page 1\n", + "Processing 745 of 1000 on page 1\n", + "Processing 746 of 1000 on page 1\n", + "Processing 747 of 1000 on page 1\n", + "Processing 748 of 1000 on page 1\n", + "Processing 749 of 1000 on page 1\n", + "Processing 750 of 1000 on page 1\n", + "Processing 751 of 1000 on page 1\n", + "Processing 752 of 1000 on page 1\n", + "Processing 753 of 1000 on page 1\n", + "Processing 754 of 1000 on page 1\n", + "Processing 755 of 1000 on page 1\n", + "Processing 756 of 1000 on page 1\n", + "Processing 757 of 1000 on page 1\n", + "Processing 758 of 1000 on page 1\n", + "Processing 759 of 1000 on page 1\n", + "Processing 760 of 1000 on page 1\n", + "Processing 761 of 1000 on page 1\n", + "Processing 762 of 1000 on page 1\n", + "Processing 763 of 1000 on page 1\n", + "Processing 764 of 1000 on page 1\n", + "Processing 765 of 1000 on page 1\n", + "Processing 766 of 1000 on page 1\n", + "Processing 767 of 1000 on page 1\n", + "Processing 768 of 1000 on page 1\n", + "Processing 769 of 1000 on page 1\n", + "Processing 770 of 1000 on page 1\n", + "Processing 771 of 1000 on page 1\n", + "Processing 772 of 1000 on page 1\n", + "Processing 773 of 1000 on page 1\n", + "Processing 774 of 1000 on page 1\n", + "Processing 775 of 1000 on page 1\n", + "Processing 776 of 1000 on page 1\n", + "Processing 777 of 1000 on page 1\n", + "Processing 778 of 1000 on page 1\n", + "Processing 779 of 1000 on page 1\n", + "Processing 780 of 1000 on page 1\n", + "Processing 781 of 1000 on page 1\n", + "Processing 782 of 1000 on page 1\n", + "Processing 783 of 1000 on page 1\n", + "Processing 784 of 1000 on page 1\n", + "Processing 785 of 1000 on page 1\n", + "Processing 786 of 1000 on page 1\n", + "Processing 787 of 1000 on page 1\n", + "Processing 788 of 1000 on page 1\n", + "Processing 789 of 1000 on page 1\n", + "Processing 790 of 1000 on page 1\n", + "Processing 791 of 1000 on page 1\n", + "Processing 792 of 1000 on page 1\n", + "Processing 793 of 1000 on page 1\n", + "Processing 794 of 1000 on page 1\n", + "Processing 795 of 1000 on page 1\n", + "Processing 796 of 1000 on page 1\n", + "Processing 797 of 1000 on page 1\n", + "Processing 798 of 1000 on page 1\n", + "Processing 799 of 1000 on page 1\n", + "Processing 800 of 1000 on page 1\n", + "Processing 801 of 1000 on page 1\n", + "Processing 802 of 1000 on page 1\n", + "Processing 803 of 1000 on page 1\n", + "Processing 804 of 1000 on page 1\n", + "Processing 805 of 1000 on page 1\n", + "Processing 806 of 1000 on page 1\n", + "Processing 807 of 1000 on page 1\n", + "Processing 808 of 1000 on page 1\n", + "Processing 809 of 1000 on page 1\n", + "Processing 810 of 1000 on page 1\n", + "Processing 811 of 1000 on page 1\n", + "Processing 812 of 1000 on page 1\n", + "Processing 813 of 1000 on page 1\n", + "Processing 814 of 1000 on page 1\n", + "Processing 815 of 1000 on page 1\n", + "Processing 816 of 1000 on page 1\n", + "Processing 817 of 1000 on page 1\n", + "Processing 818 of 1000 on page 1\n", + "Processing 819 of 1000 on page 1\n", + "Processing 820 of 1000 on page 1\n", + "Processing 821 of 1000 on page 1\n", + "Processing 822 of 1000 on page 1\n", + "Processing 823 of 1000 on page 1\n", + "Processing 824 of 1000 on page 1\n", + "Processing 825 of 1000 on page 1\n", + "Processing 826 of 1000 on page 1\n", + "Processing 827 of 1000 on page 1\n", + "Processing 828 of 1000 on page 1\n", + "Processing 829 of 1000 on page 1\n", + "Processing 830 of 1000 on page 1\n", + "Processing 831 of 1000 on page 1\n", + "Processing 832 of 1000 on page 1\n", + "Processing 833 of 1000 on page 1\n", + "Processing 834 of 1000 on page 1\n", + "Processing 835 of 1000 on page 1\n", + "Processing 836 of 1000 on page 1\n", + "Processing 837 of 1000 on page 1\n", + "Processing 838 of 1000 on page 1\n", + "Processing 839 of 1000 on page 1\n", + "Processing 840 of 1000 on page 1\n", + "Processing 841 of 1000 on page 1\n", + "Processing 842 of 1000 on page 1\n", + "Processing 843 of 1000 on page 1\n", + "Processing 844 of 1000 on page 1\n", + "Processing 845 of 1000 on page 1\n", + "Processing 846 of 1000 on page 1\n", + "Processing 847 of 1000 on page 1\n", + "Processing 848 of 1000 on page 1\n", + "Processing 849 of 1000 on page 1\n", + "Processing 850 of 1000 on page 1\n", + "Processing 851 of 1000 on page 1\n", + "Processing 852 of 1000 on page 1\n", + "Processing 853 of 1000 on page 1\n", + "Processing 854 of 1000 on page 1\n", + "Processing 855 of 1000 on page 1\n", + "Processing 856 of 1000 on page 1\n", + "Processing 857 of 1000 on page 1\n", + "Processing 858 of 1000 on page 1\n", + "Processing 859 of 1000 on page 1\n", + "Processing 860 of 1000 on page 1\n", + "Processing 861 of 1000 on page 1\n", + "Processing 862 of 1000 on page 1\n", + "Processing 863 of 1000 on page 1\n", + "Processing 864 of 1000 on page 1\n", + "Processing 865 of 1000 on page 1\n", + "Processing 866 of 1000 on page 1\n", + "Processing 867 of 1000 on page 1\n", + "Processing 868 of 1000 on page 1\n", + "Processing 869 of 1000 on page 1\n", + "Processing 870 of 1000 on page 1\n", + "Processing 871 of 1000 on page 1\n", + "Processing 872 of 1000 on page 1\n", + "Processing 873 of 1000 on page 1\n", + "Processing 874 of 1000 on page 1\n", + "Processing 875 of 1000 on page 1\n", + "Processing 876 of 1000 on page 1\n", + "Processing 877 of 1000 on page 1\n", + "Processing 878 of 1000 on page 1\n", + "Processing 879 of 1000 on page 1\n", + "Processing 880 of 1000 on page 1\n", + "Processing 881 of 1000 on page 1\n", + "Processing 882 of 1000 on page 1\n", + "Processing 883 of 1000 on page 1\n", + "Processing 884 of 1000 on page 1\n", + "Processing 885 of 1000 on page 1\n", + "Processing 886 of 1000 on page 1\n", + "Processing 887 of 1000 on page 1\n", + "Processing 888 of 1000 on page 1\n", + "Processing 889 of 1000 on page 1\n", + "Processing 890 of 1000 on page 1\n", + "Processing 891 of 1000 on page 1\n", + "Processing 892 of 1000 on page 1\n", + "Processing 893 of 1000 on page 1\n", + "Processing 894 of 1000 on page 1\n", + "Processing 895 of 1000 on page 1\n", + "Processing 896 of 1000 on page 1\n", + "Processing 897 of 1000 on page 1\n", + "Processing 898 of 1000 on page 1\n", + "Processing 899 of 1000 on page 1\n", + "Processing 900 of 1000 on page 1\n", + "Processing 901 of 1000 on page 1\n", + "Processing 902 of 1000 on page 1\n", + "Processing 903 of 1000 on page 1\n", + "Processing 904 of 1000 on page 1\n", + "Processing 905 of 1000 on page 1\n", + "Processing 906 of 1000 on page 1\n", + "Processing 907 of 1000 on page 1\n", + "Processing 908 of 1000 on page 1\n", + "Processing 909 of 1000 on page 1\n", + "Processing 910 of 1000 on page 1\n", + "Processing 911 of 1000 on page 1\n", + "Processing 912 of 1000 on page 1\n", + "Processing 913 of 1000 on page 1\n", + "Processing 914 of 1000 on page 1\n", + "Processing 915 of 1000 on page 1\n", + "Processing 916 of 1000 on page 1\n", + "Processing 917 of 1000 on page 1\n", + "Processing 918 of 1000 on page 1\n", + "Processing 919 of 1000 on page 1\n", + "Processing 920 of 1000 on page 1\n", + "Processing 921 of 1000 on page 1\n", + "Processing 922 of 1000 on page 1\n", + "Processing 923 of 1000 on page 1\n", + "Processing 924 of 1000 on page 1\n", + "Processing 925 of 1000 on page 1\n", + "Processing 926 of 1000 on page 1\n", + "Processing 927 of 1000 on page 1\n", + "Processing 928 of 1000 on page 1\n", + "Processing 929 of 1000 on page 1\n", + "Processing 930 of 1000 on page 1\n", + "Processing 931 of 1000 on page 1\n", + "Processing 932 of 1000 on page 1\n", + "Processing 933 of 1000 on page 1\n", + "Processing 934 of 1000 on page 1\n", + "Processing 935 of 1000 on page 1\n", + "Processing 936 of 1000 on page 1\n", + "Processing 937 of 1000 on page 1\n", + "Processing 938 of 1000 on page 1\n", + "Processing 939 of 1000 on page 1\n", + "Processing 940 of 1000 on page 1\n", + "Processing 941 of 1000 on page 1\n", + "Processing 942 of 1000 on page 1\n", + "Processing 943 of 1000 on page 1\n", + "Processing 944 of 1000 on page 1\n", + "Processing 945 of 1000 on page 1\n", + "Processing 946 of 1000 on page 1\n", + "Processing 947 of 1000 on page 1\n", + "Processing 948 of 1000 on page 1\n", + "Processing 949 of 1000 on page 1\n", + "Processing 950 of 1000 on page 1\n", + "Processing 951 of 1000 on page 1\n", + "Processing 952 of 1000 on page 1\n", + "Processing 953 of 1000 on page 1\n", + "Processing 954 of 1000 on page 1\n", + "Processing 955 of 1000 on page 1\n", + "Processing 956 of 1000 on page 1\n", + "Processing 957 of 1000 on page 1\n", + "Processing 958 of 1000 on page 1\n", + "Processing 959 of 1000 on page 1\n", + "Processing 960 of 1000 on page 1\n", + "Processing 961 of 1000 on page 1\n", + "Processing 962 of 1000 on page 1\n", + "Processing 963 of 1000 on page 1\n", + "Processing 964 of 1000 on page 1\n", + "Processing 965 of 1000 on page 1\n", + "Processing 966 of 1000 on page 1\n", + "Processing 967 of 1000 on page 1\n", + "Processing 968 of 1000 on page 1\n", + "Processing 969 of 1000 on page 1\n", + "Processing 970 of 1000 on page 1\n", + "Processing 971 of 1000 on page 1\n", + "Processing 972 of 1000 on page 1\n", + "Processing 973 of 1000 on page 1\n", + "Processing 974 of 1000 on page 1\n", + "Processing 975 of 1000 on page 1\n", + "Processing 976 of 1000 on page 1\n", + "Processing 977 of 1000 on page 1\n", + "Processing 978 of 1000 on page 1\n", + "Processing 979 of 1000 on page 1\n", + "Processing 980 of 1000 on page 1\n", + "Processing 981 of 1000 on page 1\n", + "Processing 982 of 1000 on page 1\n", + "Processing 983 of 1000 on page 1\n", + "Processing 984 of 1000 on page 1\n", + "Processing 985 of 1000 on page 1\n", + "Processing 986 of 1000 on page 1\n", + "Processing 987 of 1000 on page 1\n", + "Processing 988 of 1000 on page 1\n", + "Processing 989 of 1000 on page 1\n", + "Processing 990 of 1000 on page 1\n", + "Processing 991 of 1000 on page 1\n", + "Processing 992 of 1000 on page 1\n", + "Processing 993 of 1000 on page 1\n", + "Processing 994 of 1000 on page 1\n", + "Processing 995 of 1000 on page 1\n", + "Processing 996 of 1000 on page 1\n", + "Processing 997 of 1000 on page 1\n", + "Processing 998 of 1000 on page 1\n", + "Processing 999 of 1000 on page 1\n", + "Processing 1000 of 1000 on page 1\n", + "Processing 1001 of 1000 on page 2\n", + "Processing 1002 of 1000 on page 2\n", + "Processing 1003 of 1000 on page 2\n", + "Processing 1004 of 1000 on page 2\n", + "Processing 1005 of 1000 on page 2\n", + "Processing 1006 of 1000 on page 2\n", + "Processing 1007 of 1000 on page 2\n", + "Processing 1008 of 1000 on page 2\n", + "Processing 1009 of 1000 on page 2\n", + "Processing 1010 of 1000 on page 2\n", + "Processing 1011 of 1000 on page 2\n", + "Processing 1012 of 1000 on page 2\n", + "Processing 1013 of 1000 on page 2\n", + "Processing 1014 of 1000 on page 2\n", + "Processing 1015 of 1000 on page 2\n", + "Processing 1016 of 1000 on page 2\n", + "Processing 1017 of 1000 on page 2\n", + "Processing 1018 of 1000 on page 2\n", + "Processing 1019 of 1000 on page 2\n", + "Processing 1020 of 1000 on page 2\n", + "Processing 1021 of 1000 on page 2\n", + "Processing 1022 of 1000 on page 2\n", + "Processing 1023 of 1000 on page 2\n", + "Processing 1024 of 1000 on page 2\n", + "Processing 1025 of 1000 on page 2\n", + "Processing 1026 of 1000 on page 2\n", + "Processing 1027 of 1000 on page 2\n", + "Processing 1028 of 1000 on page 2\n", + "Processing 1029 of 1000 on page 2\n", + "Processing 1030 of 1000 on page 2\n", + "Processing 1031 of 1000 on page 2\n", + "Processing 1032 of 1000 on page 2\n", + "Processing 1033 of 1000 on page 2\n", + "Processing 1034 of 1000 on page 2\n", + "Processing 1035 of 1000 on page 2\n", + "Processing 1036 of 1000 on page 2\n", + "Processing 1037 of 1000 on page 2\n", + "Processing 1038 of 1000 on page 2\n", + "Processing 1039 of 1000 on page 2\n", + "Processing 1040 of 1000 on page 2\n", + "Processing 1041 of 1000 on page 2\n", + "Processing 1042 of 1000 on page 2\n", + "Processing 1043 of 1000 on page 2\n", + "Processing 1044 of 1000 on page 2\n", + "Processing 1045 of 1000 on page 2\n", + "Processing 1046 of 1000 on page 2\n", + "Processing 1047 of 1000 on page 2\n", + "Processing 1048 of 1000 on page 2\n", + "Processing 1049 of 1000 on page 2\n", + "Processing 1050 of 1000 on page 2\n", + "Processing 1051 of 1000 on page 2\n", + "Processing 1052 of 1000 on page 2\n", + "Processing 1053 of 1000 on page 2\n", + "Processing 1054 of 1000 on page 2\n", + "Processing 1055 of 1000 on page 2\n", + "Processing 1056 of 1000 on page 2\n", + "Processing 1057 of 1000 on page 2\n", + "Processing 1058 of 1000 on page 2\n", + "Processing 1059 of 1000 on page 2\n", + "Processing 1060 of 1000 on page 2\n", + "Processing 1061 of 1000 on page 2\n", + "Processing 1062 of 1000 on page 2\n", + "Processing 1063 of 1000 on page 2\n", + "Processing 1064 of 1000 on page 2\n", + "Processing 1065 of 1000 on page 2\n", + "Processing 1066 of 1000 on page 2\n", + "Processing 1067 of 1000 on page 2\n", + "Processing 1068 of 1000 on page 2\n", + "Processing 1069 of 1000 on page 2\n", + "Processing 1070 of 1000 on page 2\n", + "Processing 1071 of 1000 on page 2\n", + "Processing 1072 of 1000 on page 2\n", + "Processing 1073 of 1000 on page 2\n", + "Processing 1074 of 1000 on page 2\n", + "Processing 1075 of 1000 on page 2\n", + "Processing 1076 of 1000 on page 2\n", + "Processing 1077 of 1000 on page 2\n", + "Processing 1078 of 1000 on page 2\n", + "Processing 1079 of 1000 on page 2\n", + "Processing 1080 of 1000 on page 2\n", + "Processing 1081 of 1000 on page 2\n", + "Processing 1082 of 1000 on page 2\n", + "Processing 1083 of 1000 on page 2\n", + "Processing 1084 of 1000 on page 2\n", + "Processing 1085 of 1000 on page 2\n", + "Processing 1086 of 1000 on page 2\n", + "Processing 1087 of 1000 on page 2\n", + "Processing 1088 of 1000 on page 2\n", + "Processing 1089 of 1000 on page 2\n", + "Processing 1090 of 1000 on page 2\n", + "Processing 1091 of 1000 on page 2\n", + "Processing 1092 of 1000 on page 2\n", + "Processing 1093 of 1000 on page 2\n", + "Processing 1094 of 1000 on page 2\n", + "Processing 1095 of 1000 on page 2\n", + "Processing 1096 of 1000 on page 2\n", + "Processing 1097 of 1000 on page 2\n", + "Processing 1098 of 1000 on page 2\n", + "Processing 1099 of 1000 on page 2\n", + "Processing 1100 of 1000 on page 2\n", + "Processing 1101 of 1000 on page 2\n", + "Processing 1102 of 1000 on page 2\n", + "Processing 1103 of 1000 on page 2\n", + "Processing 1104 of 1000 on page 2\n", + "Processing 1105 of 1000 on page 2\n", + "Processing 1106 of 1000 on page 2\n", + "Processing 1107 of 1000 on page 2\n", + "Processing 1108 of 1000 on page 2\n", + "Processing 1109 of 1000 on page 2\n", + "Processing 1110 of 1000 on page 2\n", + "Processing 1111 of 1000 on page 2\n", + "Processing 1112 of 1000 on page 2\n", + "Processing 1113 of 1000 on page 2\n", + "Processing 1114 of 1000 on page 2\n", + "Processing 1115 of 1000 on page 2\n", + "Processing 1116 of 1000 on page 2\n", + "Processing 1117 of 1000 on page 2\n", + "Processing 1118 of 1000 on page 2\n", + "Processing 1119 of 1000 on page 2\n", + "Processing 1120 of 1000 on page 2\n", + "Processing 1121 of 1000 on page 2\n", + "Processing 1122 of 1000 on page 2\n", + "Processing 1123 of 1000 on page 2\n", + "Processing 1124 of 1000 on page 2\n", + "Processing 1125 of 1000 on page 2\n", + "Processing 1126 of 1000 on page 2\n", + "Processing 1127 of 1000 on page 2\n", + "Processing 1128 of 1000 on page 2\n", + "Processing 1129 of 1000 on page 2\n", + "Processing 1130 of 1000 on page 2\n", + "Processing 1131 of 1000 on page 2\n", + "Processing 1132 of 1000 on page 2\n", + "Processing 1133 of 1000 on page 2\n", + "Processing 1134 of 1000 on page 2\n", + "Processing 1135 of 1000 on page 2\n", + "Processing 1136 of 1000 on page 2\n", + "Processing 1137 of 1000 on page 2\n", + "Processing 1138 of 1000 on page 2\n", + "Processing 1139 of 1000 on page 2\n", + "Processing 1140 of 1000 on page 2\n", + "Processing 1141 of 1000 on page 2\n", + "Processing 1142 of 1000 on page 2\n", + "Processing 1143 of 1000 on page 2\n", + "Processing 1144 of 1000 on page 2\n", + "Processing 1145 of 1000 on page 2\n", + "Processing 1146 of 1000 on page 2\n", + "Processing 1147 of 1000 on page 2\n", + "Processing 1148 of 1000 on page 2\n", + "Processing 1149 of 1000 on page 2\n", + "Processing 1150 of 1000 on page 2\n", + "Processing 1151 of 1000 on page 2\n", + "Processing 1152 of 1000 on page 2\n", + "Processing 1153 of 1000 on page 2\n", + "Processing 1154 of 1000 on page 2\n", + "Processing 1155 of 1000 on page 2\n", + "Processing 1156 of 1000 on page 2\n", + "Processing 1157 of 1000 on page 2\n", + "Processing 1158 of 1000 on page 2\n", + "Processing 1159 of 1000 on page 2\n", + "Processing 1160 of 1000 on page 2\n", + "Processing 1161 of 1000 on page 2\n", + "Processing 1162 of 1000 on page 2\n", + "Processing 1163 of 1000 on page 2\n", + "Processing 1164 of 1000 on page 2\n", + "Processing 1165 of 1000 on page 2\n", + "Processing 1166 of 1000 on page 2\n", + "Processing 1167 of 1000 on page 2\n", + "Processing 1168 of 1000 on page 2\n", + "Processing 1169 of 1000 on page 2\n", + "Processing 1170 of 1000 on page 2\n", + "Processing 1171 of 1000 on page 2\n", + "Processing 1172 of 1000 on page 2\n", + "Processing 1173 of 1000 on page 2\n", + "Processing 1174 of 1000 on page 2\n", + "Processing 1175 of 1000 on page 2\n", + "Processing 1176 of 1000 on page 2\n", + "Processing 1177 of 1000 on page 2\n", + "Processing 1178 of 1000 on page 2\n", + "Processing 1179 of 1000 on page 2\n", + "Processing 1180 of 1000 on page 2\n", + "Processing 1181 of 1000 on page 2\n", + "Processing 1182 of 1000 on page 2\n", + "Processing 1183 of 1000 on page 2\n", + "Processing 1184 of 1000 on page 2\n", + "Processing 1185 of 1000 on page 2\n", + "Processing 1186 of 1000 on page 2\n", + "Processing 1187 of 1000 on page 2\n", + "Processing 1188 of 1000 on page 2\n", + "Processing 1189 of 1000 on page 2\n", + "Processing 1190 of 1000 on page 2\n", + "Processing 1191 of 1000 on page 2\n", + "Processing 1192 of 1000 on page 2\n", + "Processing 1193 of 1000 on page 2\n", + "Processing 1194 of 1000 on page 2\n", + "Processing 1195 of 1000 on page 2\n", + "Processing 1196 of 1000 on page 2\n", + "Processing 1197 of 1000 on page 2\n", + "Processing 1198 of 1000 on page 2\n", + "Processing 1199 of 1000 on page 2\n", + "Processing 1200 of 1000 on page 2\n", + "Processing 1201 of 1000 on page 2\n", + "Processing 1202 of 1000 on page 2\n", + "Processing 1203 of 1000 on page 2\n", + "Processing 1204 of 1000 on page 2\n", + "Processing 1205 of 1000 on page 2\n", + "Processing 1206 of 1000 on page 2\n", + "Processing 1207 of 1000 on page 2\n", + "Processing 1208 of 1000 on page 2\n", + "Processing 1209 of 1000 on page 2\n", + "Processing 1210 of 1000 on page 2\n", + "Processing 1211 of 1000 on page 2\n", + "Processing 1212 of 1000 on page 2\n", + "Processing 1213 of 1000 on page 2\n", + "Processing 1214 of 1000 on page 2\n", + "Processing 1215 of 1000 on page 2\n", + "Processing 1216 of 1000 on page 2\n", + "Processing 1217 of 1000 on page 2\n", + "Processing 1218 of 1000 on page 2\n", + "Processing 1219 of 1000 on page 2\n", + "Processing 1220 of 1000 on page 2\n", + "Processing 1221 of 1000 on page 2\n", + "Processing 1222 of 1000 on page 2\n", + "Processing 1223 of 1000 on page 2\n", + "Processing 1224 of 1000 on page 2\n", + "Processing 1225 of 1000 on page 2\n", + "Processing 1226 of 1000 on page 2\n", + "Processing 1227 of 1000 on page 2\n", + "Processing 1228 of 1000 on page 2\n", + "Processing 1229 of 1000 on page 2\n", + "Processing 1230 of 1000 on page 2\n", + "Processing 1231 of 1000 on page 2\n", + "Processing 1232 of 1000 on page 2\n", + "Processing 1233 of 1000 on page 2\n", + "Processing 1234 of 1000 on page 2\n", + "Processing 1235 of 1000 on page 2\n", + "Processing 1236 of 1000 on page 2\n", + "Processing 1237 of 1000 on page 2\n", + "Processing 1238 of 1000 on page 2\n", + "Processing 1239 of 1000 on page 2\n", + "Processing 1240 of 1000 on page 2\n", + "Processing 1241 of 1000 on page 2\n", + "Processing 1242 of 1000 on page 2\n", + "Processing 1243 of 1000 on page 2\n", + "Processing 1244 of 1000 on page 2\n", + "Processing 1245 of 1000 on page 2\n", + "Processing 1246 of 1000 on page 2\n", + "Processing 1247 of 1000 on page 2\n", + "Processing 1248 of 1000 on page 2\n", + "Processing 1249 of 1000 on page 2\n", + "Processing 1250 of 1000 on page 2\n", + "Processing 1251 of 1000 on page 2\n", + "Processing 1252 of 1000 on page 2\n", + "Processing 1253 of 1000 on page 2\n", + "Processing 1254 of 1000 on page 2\n", + "Processing 1255 of 1000 on page 2\n", + "Processing 1256 of 1000 on page 2\n", + "Processing 1257 of 1000 on page 2\n", + "Processing 1258 of 1000 on page 2\n", + "Processing 1259 of 1000 on page 2\n", + "Processing 1260 of 1000 on page 2\n", + "Processing 1261 of 1000 on page 2\n", + "Processing 1262 of 1000 on page 2\n", + "Processing 1263 of 1000 on page 2\n", + "Processing 1264 of 1000 on page 2\n", + "Processing 1265 of 1000 on page 2\n", + "Processing 1266 of 1000 on page 2\n", + "Processing 1267 of 1000 on page 2\n", + "Processing 1268 of 1000 on page 2\n", + "Processing 1269 of 1000 on page 2\n", + "Processing 1270 of 1000 on page 2\n", + "Processing 1271 of 1000 on page 2\n", + "Processing 1272 of 1000 on page 2\n", + "Processing 1273 of 1000 on page 2\n", + "Processing 1274 of 1000 on page 2\n", + "Processing 1275 of 1000 on page 2\n", + "Processing 1276 of 1000 on page 2\n", + "Processing 1277 of 1000 on page 2\n", + "Processing 1278 of 1000 on page 2\n", + "Processing 1279 of 1000 on page 2\n", + "Processing 1280 of 1000 on page 2\n", + "Processing 1281 of 1000 on page 2\n", + "Processing 1282 of 1000 on page 2\n", + "Processing 1283 of 1000 on page 2\n", + "Processing 1284 of 1000 on page 2\n", + "Processing 1285 of 1000 on page 2\n", + "Processing 1286 of 1000 on page 2\n", + "Processing 1287 of 1000 on page 2\n", + "Processing 1288 of 1000 on page 2\n", + "Processing 1289 of 1000 on page 2\n", + "Processing 1290 of 1000 on page 2\n", + "Processing 1291 of 1000 on page 2\n", + "Processing 1292 of 1000 on page 2\n", + "Processing 1293 of 1000 on page 2\n", + "Processing 1294 of 1000 on page 2\n", + "Processing 1295 of 1000 on page 2\n", + "Processing 1296 of 1000 on page 2\n", + "Processing 1297 of 1000 on page 2\n", + "Processing 1298 of 1000 on page 2\n", + "Processing 1299 of 1000 on page 2\n", + "Processing 1300 of 1000 on page 2\n", + "Processing 1301 of 1000 on page 2\n", + "Processing 1302 of 1000 on page 2\n", + "Processing 1303 of 1000 on page 2\n", + "Processing 1304 of 1000 on page 2\n", + "Processing 1305 of 1000 on page 2\n", + "Processing 1306 of 1000 on page 2\n", + "Processing 1307 of 1000 on page 2\n", + "Processing 1308 of 1000 on page 2\n", + "Processing 1309 of 1000 on page 2\n", + "Processing 1310 of 1000 on page 2\n", + "Processing 1311 of 1000 on page 2\n", + "Processing 1312 of 1000 on page 2\n", + "Processing 1313 of 1000 on page 2\n", + "Processing 1314 of 1000 on page 2\n", + "Processing 1315 of 1000 on page 2\n", + "Processing 1316 of 1000 on page 2\n", + "Processing 1317 of 1000 on page 2\n", + "Processing 1318 of 1000 on page 2\n", + "Processing 1319 of 1000 on page 2\n", + "Processing 1320 of 1000 on page 2\n", + "Processing 1321 of 1000 on page 2\n", + "Processing 1322 of 1000 on page 2\n", + "Processing 1323 of 1000 on page 2\n", + "Processing 1324 of 1000 on page 2\n", + "Processing 1325 of 1000 on page 2\n", + "Processing 1326 of 1000 on page 2\n", + "Processing 1327 of 1000 on page 2\n", + "Processing 1328 of 1000 on page 2\n", + "Processing 1329 of 1000 on page 2\n", + "Processing 1330 of 1000 on page 2\n", + "Processing 1331 of 1000 on page 2\n", + "Processing 1332 of 1000 on page 2\n", + "Processing 1333 of 1000 on page 2\n", + "Processing 1334 of 1000 on page 2\n", + "Processing 1335 of 1000 on page 2\n", + "Processing 1336 of 1000 on page 2\n", + "Processing 1337 of 1000 on page 2\n", + "Processing 1338 of 1000 on page 2\n", + "Processing 1339 of 1000 on page 2\n", + "Processing 1340 of 1000 on page 2\n", + "Processing 1341 of 1000 on page 2\n", + "Processing 1342 of 1000 on page 2\n", + "Processing 1343 of 1000 on page 2\n", + "Processing 1344 of 1000 on page 2\n", + "Processing 1345 of 1000 on page 2\n", + "Processing 1346 of 1000 on page 2\n", + "Processing 1347 of 1000 on page 2\n", + "Processing 1348 of 1000 on page 2\n", + "Processing 1349 of 1000 on page 2\n", + "Processing 1350 of 1000 on page 2\n", + "Processing 1351 of 1000 on page 2\n", + "Processing 1352 of 1000 on page 2\n", + "Processing 1353 of 1000 on page 2\n", + "Processing 1354 of 1000 on page 2\n", + "Processing 1355 of 1000 on page 2\n", + "Processing 1356 of 1000 on page 2\n", + "Processing 1357 of 1000 on page 2\n", + "Processing 1358 of 1000 on page 2\n", + "Processing 1359 of 1000 on page 2\n", + "Processing 1360 of 1000 on page 2\n", + "Processing 1361 of 1000 on page 2\n", + "Processing 1362 of 1000 on page 2\n", + "Processing 1363 of 1000 on page 2\n", + "Processing 1364 of 1000 on page 2\n", + "Processing 1365 of 1000 on page 2\n", + "Processing 1366 of 1000 on page 2\n", + "Processing 1367 of 1000 on page 2\n", + "Processing 1368 of 1000 on page 2\n", + "Processing 1369 of 1000 on page 2\n", + "Processing 1370 of 1000 on page 2\n", + "Processing 1371 of 1000 on page 2\n", + "Processing 1372 of 1000 on page 2\n", + "Processing 1373 of 1000 on page 2\n", + "Processing 1374 of 1000 on page 2\n", + "Processing 1375 of 1000 on page 2\n", + "Processing 1376 of 1000 on page 2\n", + "Processing 1377 of 1000 on page 2\n", + "Processing 1378 of 1000 on page 2\n", + "Processing 1379 of 1000 on page 2\n", + "Processing 1380 of 1000 on page 2\n", + "Processing 1381 of 1000 on page 2\n", + "Processing 1382 of 1000 on page 2\n", + "Processing 1383 of 1000 on page 2\n", + "Processing 1384 of 1000 on page 2\n", + "Processing 1385 of 1000 on page 2\n", + "Processing 1386 of 1000 on page 2\n", + "Processing 1387 of 1000 on page 2\n", + "Processing 1388 of 1000 on page 2\n", + "Processing 1389 of 1000 on page 2\n", + "Processing 1390 of 1000 on page 2\n", + "Processing 1391 of 1000 on page 2\n", + "Processing 1392 of 1000 on page 2\n", + "Processing 1393 of 1000 on page 2\n", + "Processing 1394 of 1000 on page 2\n", + "Processing 1395 of 1000 on page 2\n", + "Processing 1396 of 1000 on page 2\n", + "Processing 1397 of 1000 on page 2\n", + "Processing 1398 of 1000 on page 2\n", + "Processing 1399 of 1000 on page 2\n", + "Processing 1400 of 1000 on page 2\n", + "Processing 1401 of 1000 on page 2\n", + "Processing 1402 of 1000 on page 2\n", + "Processing 1403 of 1000 on page 2\n", + "Processing 1404 of 1000 on page 2\n", + "Processing 1405 of 1000 on page 2\n", + "Processing 1406 of 1000 on page 2\n", + "Processing 1407 of 1000 on page 2\n", + "Processing 1408 of 1000 on page 2\n", + "Processing 1409 of 1000 on page 2\n", + "Processing 1410 of 1000 on page 2\n", + "Processing 1411 of 1000 on page 2\n", + "Processing 1412 of 1000 on page 2\n", + "Processing 1413 of 1000 on page 2\n", + "Processing 1414 of 1000 on page 2\n", + "Processing 1415 of 1000 on page 2\n", + "Processing 1416 of 1000 on page 2\n", + "Processing 1417 of 1000 on page 2\n", + "Processing 1418 of 1000 on page 2\n", + "Processing 1419 of 1000 on page 2\n", + "Processing 1420 of 1000 on page 2\n", + "Processing 1421 of 1000 on page 2\n", + "Processing 1422 of 1000 on page 2\n", + "Processing 1423 of 1000 on page 2\n", + "Processing 1424 of 1000 on page 2\n", + "Processing 1425 of 1000 on page 2\n", + "Processing 1426 of 1000 on page 2\n", + "Processing 1427 of 1000 on page 2\n", + "Processing 1428 of 1000 on page 2\n", + "Processing 1429 of 1000 on page 2\n", + "Processing 1430 of 1000 on page 2\n", + "Processing 1431 of 1000 on page 2\n", + "Processing 1432 of 1000 on page 2\n", + "Processing 1433 of 1000 on page 2\n", + "Processing 1434 of 1000 on page 2\n", + "Processing 1435 of 1000 on page 2\n", + "Processing 1436 of 1000 on page 2\n", + "Processing 1437 of 1000 on page 2\n", + "Processing 1438 of 1000 on page 2\n", + "Processing 1439 of 1000 on page 2\n", + "Processing 1440 of 1000 on page 2\n", + "Processing 1441 of 1000 on page 2\n", + "Processing 1442 of 1000 on page 2\n", + "Processing 1443 of 1000 on page 2\n", + "Processing 1444 of 1000 on page 2\n", + "Processing 1445 of 1000 on page 2\n", + "Processing 1446 of 1000 on page 2\n", + "Processing 1447 of 1000 on page 2\n", + "Processing 1448 of 1000 on page 2\n", + "Processing 1449 of 1000 on page 2\n", + "Processing 1450 of 1000 on page 2\n", + "Processing 1451 of 1000 on page 2\n", + "Processing 1452 of 1000 on page 2\n", + "Processing 1453 of 1000 on page 2\n", + "Processing 1454 of 1000 on page 2\n", + "Processing 1455 of 1000 on page 2\n", + "Processing 1456 of 1000 on page 2\n", + "Processing 1457 of 1000 on page 2\n", + "Processing 1458 of 1000 on page 2\n", + "Processing 1459 of 1000 on page 2\n", + "Processing 1460 of 1000 on page 2\n", + "Processing 1461 of 1000 on page 2\n", + "Processing 1462 of 1000 on page 2\n", + "Processing 1463 of 1000 on page 2\n", + "Processing 1464 of 1000 on page 2\n", + "Processing 1465 of 1000 on page 2\n", + "Processing 1466 of 1000 on page 2\n", + "Processing 1467 of 1000 on page 2\n", + "Processing 1468 of 1000 on page 2\n", + "Processing 1469 of 1000 on page 2\n", + "Processing 1470 of 1000 on page 2\n", + "Processing 1471 of 1000 on page 2\n", + "Processing 1472 of 1000 on page 2\n", + "Processing 1473 of 1000 on page 2\n", + "Processing 1474 of 1000 on page 2\n", + "Processing 1475 of 1000 on page 2\n", + "Processing 1476 of 1000 on page 2\n", + "Processing 1477 of 1000 on page 2\n", + "Processing 1478 of 1000 on page 2\n", + "Processing 1479 of 1000 on page 2\n", + "Processing 1480 of 1000 on page 2\n", + "Processing 1481 of 1000 on page 2\n", + "Processing 1482 of 1000 on page 2\n", + "Processing 1483 of 1000 on page 2\n", + "Processing 1484 of 1000 on page 2\n", + "Processing 1485 of 1000 on page 2\n", + "Processing 1486 of 1000 on page 2\n", + "Processing 1487 of 1000 on page 2\n", + "Processing 1488 of 1000 on page 2\n", + "Processing 1489 of 1000 on page 2\n", + "Processing 1490 of 1000 on page 2\n", + "Processing 1491 of 1000 on page 2\n", + "Processing 1492 of 1000 on page 2\n", + "Processing 1493 of 1000 on page 2\n", + "Processing 1494 of 1000 on page 2\n", + "Processing 1495 of 1000 on page 2\n", + "Processing 1496 of 1000 on page 2\n", + "Processing 1497 of 1000 on page 2\n", + "Processing 1498 of 1000 on page 2\n", + "Processing 1499 of 1000 on page 2\n", + "Processing 1500 of 1000 on page 2\n", + "Processing 1501 of 1000 on page 2\n", + "Processing 1502 of 1000 on page 2\n", + "Processing 1503 of 1000 on page 2\n", + "Processing 1504 of 1000 on page 2\n", + "Processing 1505 of 1000 on page 2\n", + "Processing 1506 of 1000 on page 2\n", + "Processing 1507 of 1000 on page 2\n", + "Processing 1508 of 1000 on page 2\n", + "Processing 1509 of 1000 on page 2\n", + "Processing 1510 of 1000 on page 2\n", + "Processing 1511 of 1000 on page 2\n", + "Processing 1512 of 1000 on page 2\n", + "Processing 1513 of 1000 on page 2\n", + "Processing 1514 of 1000 on page 2\n", + "Processing 1515 of 1000 on page 2\n", + "Processing 1516 of 1000 on page 2\n", + "Processing 1517 of 1000 on page 2\n", + "Processing 1518 of 1000 on page 2\n", + "Processing 1519 of 1000 on page 2\n", + "Processing 1520 of 1000 on page 2\n", + "Processing 1521 of 1000 on page 2\n", + "Processing 1522 of 1000 on page 2\n", + "Processing 1523 of 1000 on page 2\n", + "Processing 1524 of 1000 on page 2\n", + "Processing 1525 of 1000 on page 2\n", + "Processing 1526 of 1000 on page 2\n", + "Processing 1527 of 1000 on page 2\n", + "Processing 1528 of 1000 on page 2\n", + "Processing 1529 of 1000 on page 2\n", + "Processing 1530 of 1000 on page 2\n", + "Processing 1531 of 1000 on page 2\n", + "Processing 1532 of 1000 on page 2\n", + "Processing 1533 of 1000 on page 2\n", + "Processing 1534 of 1000 on page 2\n", + "Processing 1535 of 1000 on page 2\n", + "Processing 1536 of 1000 on page 2\n", + "Processing 1537 of 1000 on page 2\n", + "Processing 1538 of 1000 on page 2\n", + "Processing 1539 of 1000 on page 2\n", + "Processing 1540 of 1000 on page 2\n", + "Processing 1541 of 1000 on page 2\n", + "Processing 1542 of 1000 on page 2\n", + "Processing 1543 of 1000 on page 2\n", + "Processing 1544 of 1000 on page 2\n", + "Processing 1545 of 1000 on page 2\n", + "Processing 1546 of 1000 on page 2\n", + "Processing 1547 of 1000 on page 2\n", + "Processing 1548 of 1000 on page 2\n", + "Processing 1549 of 1000 on page 2\n", + "Processing 1550 of 1000 on page 2\n", + "Processing 1551 of 1000 on page 2\n", + "Processing 1552 of 1000 on page 2\n", + "Processing 1553 of 1000 on page 2\n", + "Processing 1554 of 1000 on page 2\n", + "Processing 1555 of 1000 on page 2\n", + "Processing 1556 of 1000 on page 2\n", + "Processing 1557 of 1000 on page 2\n", + "Processing 1558 of 1000 on page 2\n", + "Processing 1559 of 1000 on page 2\n", + "Processing 1560 of 1000 on page 2\n", + "Processing 1561 of 1000 on page 2\n", + "Processing 1562 of 1000 on page 2\n", + "Processing 1563 of 1000 on page 2\n", + "Processing 1564 of 1000 on page 2\n", + "Processing 1565 of 1000 on page 2\n", + "Processing 1566 of 1000 on page 2\n", + "Processing 1567 of 1000 on page 2\n", + "Processing 1568 of 1000 on page 2\n", + "Processing 1569 of 1000 on page 2\n", + "Processing 1570 of 1000 on page 2\n", + "Processing 1571 of 1000 on page 2\n", + "Processing 1572 of 1000 on page 2\n", + "Processing 1573 of 1000 on page 2\n", + "Processing 1574 of 1000 on page 2\n", + "Processing 1575 of 1000 on page 2\n", + "Processing 1576 of 1000 on page 2\n", + "Processing 1577 of 1000 on page 2\n", + "Processing 1578 of 1000 on page 2\n", + "Processing 1579 of 1000 on page 2\n", + "Processing 1580 of 1000 on page 2\n", + "Processing 1581 of 1000 on page 2\n", + "Processing 1582 of 1000 on page 2\n", + "Processing 1583 of 1000 on page 2\n", + "Processing 1584 of 1000 on page 2\n", + "Processing 1585 of 1000 on page 2\n", + "Processing 1586 of 1000 on page 2\n", + "Processing 1587 of 1000 on page 2\n", + "Processing 1588 of 1000 on page 2\n", + "Processing 1589 of 1000 on page 2\n", + "Processing 1590 of 1000 on page 2\n", + "Processing 1591 of 1000 on page 2\n", + "Processing 1592 of 1000 on page 2\n", + "Processing 1593 of 1000 on page 2\n", + "Processing 1594 of 1000 on page 2\n", + "Processing 1595 of 1000 on page 2\n", + "Processing 1596 of 1000 on page 2\n", + "Processing 1597 of 1000 on page 2\n", + "Processing 1598 of 1000 on page 2\n", + "Processing 1599 of 1000 on page 2\n", + "Processing 1600 of 1000 on page 2\n", + "Processing 1601 of 1000 on page 2\n", + "Processing 1602 of 1000 on page 2\n", + "Processing 1603 of 1000 on page 2\n", + "Processing 1604 of 1000 on page 2\n", + "Processing 1605 of 1000 on page 2\n", + "Processing 1606 of 1000 on page 2\n", + "Processing 1607 of 1000 on page 2\n", + "Processing 1608 of 1000 on page 2\n", + "Processing 1609 of 1000 on page 2\n", + "Processing 1610 of 1000 on page 2\n", + "Processing 1611 of 1000 on page 2\n", + "Processing 1612 of 1000 on page 2\n", + "Processing 1613 of 1000 on page 2\n", + "Processing 1614 of 1000 on page 2\n", + "Processing 1615 of 1000 on page 2\n", + "Processing 1616 of 1000 on page 2\n", + "Processing 1617 of 1000 on page 2\n", + "Processing 1618 of 1000 on page 2\n", + "Processing 1619 of 1000 on page 2\n", + "Processing 1620 of 1000 on page 2\n", + "Processing 1621 of 1000 on page 2\n", + "Processing 1622 of 1000 on page 2\n", + "Processing 1623 of 1000 on page 2\n", + "Processing 1624 of 1000 on page 2\n", + "Processing 1625 of 1000 on page 2\n", + "Processing 1626 of 1000 on page 2\n", + "Processing 1627 of 1000 on page 2\n", + "Processing 1628 of 1000 on page 2\n", + "Processing 1629 of 1000 on page 2\n", + "Processing 1630 of 1000 on page 2\n", + "Processing 1631 of 1000 on page 2\n", + "Processing 1632 of 1000 on page 2\n", + "Processing 1633 of 1000 on page 2\n", + "Processing 1634 of 1000 on page 2\n", + "Processing 1635 of 1000 on page 2\n", + "Processing 1636 of 1000 on page 2\n", + "Processing 1637 of 1000 on page 2\n", + "Processing 1638 of 1000 on page 2\n", + "Processing 1639 of 1000 on page 2\n", + "Processing 1640 of 1000 on page 2\n", + "Processing 1641 of 1000 on page 2\n", + "Processing 1642 of 1000 on page 2\n", + "Processing 1643 of 1000 on page 2\n", + "Processing 1644 of 1000 on page 2\n", + "Processing 1645 of 1000 on page 2\n", + "Processing 1646 of 1000 on page 2\n", + "Processing 1647 of 1000 on page 2\n", + "Processing 1648 of 1000 on page 2\n", + "Processing 1649 of 1000 on page 2\n", + "Processing 1650 of 1000 on page 2\n", + "Processing 1651 of 1000 on page 2\n", + "Processing 1652 of 1000 on page 2\n", + "Processing 1653 of 1000 on page 2\n", + "Processing 1654 of 1000 on page 2\n", + "Processing 1655 of 1000 on page 2\n", + "Processing 1656 of 1000 on page 2\n", + "Processing 1657 of 1000 on page 2\n", + "Processing 1658 of 1000 on page 2\n", + "Processing 1659 of 1000 on page 2\n", + "Processing 1660 of 1000 on page 2\n", + "Processing 1661 of 1000 on page 2\n", + "Processing 1662 of 1000 on page 2\n", + "Processing 1663 of 1000 on page 2\n", + "Processing 1664 of 1000 on page 2\n", + "Processing 1665 of 1000 on page 2\n", + "Processing 1666 of 1000 on page 2\n", + "Processing 1667 of 1000 on page 2\n", + "Processing 1668 of 1000 on page 2\n", + "Processing 1669 of 1000 on page 2\n", + "Processing 1670 of 1000 on page 2\n", + "Processing 1671 of 1000 on page 2\n", + "Processing 1672 of 1000 on page 2\n", + "Processing 1673 of 1000 on page 2\n", + "Processing 1674 of 1000 on page 2\n", + "Processing 1675 of 1000 on page 2\n", + "Processing 1676 of 1000 on page 2\n", + "Processing 1677 of 1000 on page 2\n", + "Processing 1678 of 1000 on page 2\n", + "Processing 1679 of 1000 on page 2\n", + "Processing 1680 of 1000 on page 2\n", + "Processing 1681 of 1000 on page 2\n", + "Processing 1682 of 1000 on page 2\n", + "Processing 1683 of 1000 on page 2\n", + "Processing 1684 of 1000 on page 2\n", + "Processing 1685 of 1000 on page 2\n", + "Processing 1686 of 1000 on page 2\n", + "Processing 1687 of 1000 on page 2\n", + "Processing 1688 of 1000 on page 2\n", + "Processing 1689 of 1000 on page 2\n", + "Processing 1690 of 1000 on page 2\n", + "Processing 1691 of 1000 on page 2\n", + "Processing 1692 of 1000 on page 2\n", + "Processing 1693 of 1000 on page 2\n", + "Processing 1694 of 1000 on page 2\n", + "Processing 1695 of 1000 on page 2\n", + "Processing 1696 of 1000 on page 2\n", + "Processing 1697 of 1000 on page 2\n", + "Processing 1698 of 1000 on page 2\n", + "Processing 1699 of 1000 on page 2\n", + "Processing 1700 of 1000 on page 2\n", + "Processing 1701 of 1000 on page 2\n", + "Processing 1702 of 1000 on page 2\n", + "Processing 1703 of 1000 on page 2\n", + "Processing 1704 of 1000 on page 2\n", + "Processing 1705 of 1000 on page 2\n", + "Processing 1706 of 1000 on page 2\n", + "Processing 1707 of 1000 on page 2\n", + "Processing 1708 of 1000 on page 2\n", + "Processing 1709 of 1000 on page 2\n", + "Processing 1710 of 1000 on page 2\n", + "Processing 1711 of 1000 on page 2\n", + "Processing 1712 of 1000 on page 2\n", + "Processing 1713 of 1000 on page 2\n", + "Processing 1714 of 1000 on page 2\n", + "Processing 1715 of 1000 on page 2\n", + "Processing 1716 of 1000 on page 2\n", + "Processing 1717 of 1000 on page 2\n", + "Processing 1718 of 1000 on page 2\n", + "Processing 1719 of 1000 on page 2\n", + "Processing 1720 of 1000 on page 2\n", + "Processing 1721 of 1000 on page 2\n", + "Processing 1722 of 1000 on page 2\n", + "Processing 1723 of 1000 on page 2\n", + "Processing 1724 of 1000 on page 2\n", + "Processing 1725 of 1000 on page 2\n", + "Processing 1726 of 1000 on page 2\n", + "Processing 1727 of 1000 on page 2\n", + "Processing 1728 of 1000 on page 2\n", + "Processing 1729 of 1000 on page 2\n", + "Processing 1730 of 1000 on page 2\n", + "Processing 1731 of 1000 on page 2\n", + "Processing 1732 of 1000 on page 2\n", + "Processing 1733 of 1000 on page 2\n", + "Processing 1734 of 1000 on page 2\n", + "Processing 1735 of 1000 on page 2\n", + "Processing 1736 of 1000 on page 2\n", + "Processing 1737 of 1000 on page 2\n", + "Processing 1738 of 1000 on page 2\n", + "Processing 1739 of 1000 on page 2\n", + "Processing 1740 of 1000 on page 2\n", + "Processing 1741 of 1000 on page 2\n", + "Processing 1742 of 1000 on page 2\n", + "Processing 1743 of 1000 on page 2\n", + "Processing 1744 of 1000 on page 2\n", + "Processing 1745 of 1000 on page 2\n", + "Processing 1746 of 1000 on page 2\n", + "Processing 1747 of 1000 on page 2\n", + "Processing 1748 of 1000 on page 2\n", + "Processing 1749 of 1000 on page 2\n", + "Processing 1750 of 1000 on page 2\n", + "Processing 1751 of 1000 on page 2\n", + "Processing 1752 of 1000 on page 2\n", + "Processing 1753 of 1000 on page 2\n", + "Processing 1754 of 1000 on page 2\n", + "Processing 1755 of 1000 on page 2\n", + "Processing 1756 of 1000 on page 2\n", + "Processing 1757 of 1000 on page 2\n", + "Processing 1758 of 1000 on page 2\n", + "Processing 1759 of 1000 on page 2\n", + "Processing 1760 of 1000 on page 2\n", + "Processing 1761 of 1000 on page 2\n", + "Processing 1762 of 1000 on page 2\n", + "Processing 1763 of 1000 on page 2\n", + "Processing 1764 of 1000 on page 2\n", + "Processing 1765 of 1000 on page 2\n", + "Processing 1766 of 1000 on page 2\n", + "Processing 1767 of 1000 on page 2\n", + "Processing 1768 of 1000 on page 2\n", + "Processing 1769 of 1000 on page 2\n", + "Processing 1770 of 1000 on page 2\n", + "Processing 1771 of 1000 on page 2\n", + "Processing 1772 of 1000 on page 2\n", + "Processing 1773 of 1000 on page 2\n", + "Processing 1774 of 1000 on page 2\n", + "Processing 1775 of 1000 on page 2\n", + "Processing 1776 of 1000 on page 2\n", + "Processing 1777 of 1000 on page 2\n", + "Processing 1778 of 1000 on page 2\n", + "Processing 1779 of 1000 on page 2\n", + "Processing 1780 of 1000 on page 2\n", + "Processing 1781 of 1000 on page 2\n", + "Processing 1782 of 1000 on page 2\n", + "Processing 1783 of 1000 on page 2\n", + "Processing 1784 of 1000 on page 2\n", + "Processing 1785 of 1000 on page 2\n", + "Processing 1786 of 1000 on page 2\n", + "Processing 1787 of 1000 on page 2\n", + "Processing 1788 of 1000 on page 2\n", + "Processing 1789 of 1000 on page 2\n", + "Processing 1790 of 1000 on page 2\n", + "Processing 1791 of 1000 on page 2\n", + "Processing 1792 of 1000 on page 2\n", + "Processing 1793 of 1000 on page 2\n", + "Processing 1794 of 1000 on page 2\n", + "Processing 1795 of 1000 on page 2\n", + "Processing 1796 of 1000 on page 2\n", + "Processing 1797 of 1000 on page 2\n", + "Processing 1798 of 1000 on page 2\n", + "Processing 1799 of 1000 on page 2\n", + "Processing 1800 of 1000 on page 2\n", + "Processing 1801 of 1000 on page 2\n", + "Processing 1802 of 1000 on page 2\n", + "Processing 1803 of 1000 on page 2\n", + "Processing 1804 of 1000 on page 2\n", + "Processing 1805 of 1000 on page 2\n", + "Processing 1806 of 1000 on page 2\n", + "Processing 1807 of 1000 on page 2\n", + "Processing 1808 of 1000 on page 2\n", + "Processing 1809 of 1000 on page 2\n", + "Processing 1810 of 1000 on page 2\n", + "Processing 1811 of 1000 on page 2\n", + "Processing 1812 of 1000 on page 2\n", + "Processing 1813 of 1000 on page 2\n", + "Processing 1814 of 1000 on page 2\n", + "Processing 1815 of 1000 on page 2\n", + "Processing 1816 of 1000 on page 2\n", + "Processing 1817 of 1000 on page 2\n", + "Processing 1818 of 1000 on page 2\n", + "Processing 1819 of 1000 on page 2\n", + "Processing 1820 of 1000 on page 2\n", + "Processing 1821 of 1000 on page 2\n", + "Processing 1822 of 1000 on page 2\n", + "Processing 1823 of 1000 on page 2\n", + "Processing 1824 of 1000 on page 2\n", + "Processing 1825 of 1000 on page 2\n", + "Processing 1826 of 1000 on page 2\n", + "Processing 1827 of 1000 on page 2\n", + "Processing 1828 of 1000 on page 2\n", + "Processing 1829 of 1000 on page 2\n", + "Processing 1830 of 1000 on page 2\n", + "Processing 1831 of 1000 on page 2\n", + "Processing 1832 of 1000 on page 2\n", + "Processing 1833 of 1000 on page 2\n", + "Processing 1834 of 1000 on page 2\n", + "Processing 1835 of 1000 on page 2\n", + "Processing 1836 of 1000 on page 2\n", + "Processing 1837 of 1000 on page 2\n", + "Processing 1838 of 1000 on page 2\n", + "Processing 1839 of 1000 on page 2\n", + "Processing 1840 of 1000 on page 2\n", + "Processing 1841 of 1000 on page 2\n", + "Processing 1842 of 1000 on page 2\n", + "Processing 1843 of 1000 on page 2\n", + "Processing 1844 of 1000 on page 2\n", + "Processing 1845 of 1000 on page 2\n", + "Processing 1846 of 1000 on page 2\n", + "Processing 1847 of 1000 on page 2\n", + "Processing 1848 of 1000 on page 2\n", + "Processing 1849 of 1000 on page 2\n", + "Processing 1850 of 1000 on page 2\n", + "Processing 1851 of 1000 on page 2\n", + "Processing 1852 of 1000 on page 2\n", + "Processing 1853 of 1000 on page 2\n", + "Processing 1854 of 1000 on page 2\n", + "Processing 1855 of 1000 on page 2\n", + "Processing 1856 of 1000 on page 2\n", + "Processing 1857 of 1000 on page 2\n", + "Processing 1858 of 1000 on page 2\n", + "Processing 1859 of 1000 on page 2\n", + "Processing 1860 of 1000 on page 2\n", + "Processing 1861 of 1000 on page 2\n", + "Processing 1862 of 1000 on page 2\n", + "Processing 1863 of 1000 on page 2\n", + "Processing 1864 of 1000 on page 2\n", + "Processing 1865 of 1000 on page 2\n", + "Processing 1866 of 1000 on page 2\n", + "Processing 1867 of 1000 on page 2\n", + "Processing 1868 of 1000 on page 2\n", + "Processing 1869 of 1000 on page 2\n", + "Processing 1870 of 1000 on page 2\n", + "Processing 1871 of 1000 on page 2\n", + "Processing 1872 of 1000 on page 2\n", + "Processing 1873 of 1000 on page 2\n", + "Processing 1874 of 1000 on page 2\n", + "Processing 1875 of 1000 on page 2\n", + "Processing 1876 of 1000 on page 2\n", + "Processing 1877 of 1000 on page 2\n", + "Processing 1878 of 1000 on page 2\n", + "Processing 1879 of 1000 on page 2\n", + "Processing 1880 of 1000 on page 2\n", + "Processing 1881 of 1000 on page 2\n", + "Processing 1882 of 1000 on page 2\n", + "Processing 1883 of 1000 on page 2\n", + "Processing 1884 of 1000 on page 2\n", + "Processing 1885 of 1000 on page 2\n", + "Processing 1886 of 1000 on page 2\n", + "Processing 1887 of 1000 on page 2\n", + "Processing 1888 of 1000 on page 2\n", + "Processing 1889 of 1000 on page 2\n", + "Processing 1890 of 1000 on page 2\n", + "Processing 1891 of 1000 on page 2\n", + "Processing 1892 of 1000 on page 2\n", + "Processing 1893 of 1000 on page 2\n", + "Processing 1894 of 1000 on page 2\n", + "Processing 1895 of 1000 on page 2\n", + "Processing 1896 of 1000 on page 2\n", + "Processing 1897 of 1000 on page 2\n", + "Processing 1898 of 1000 on page 2\n", + "Processing 1899 of 1000 on page 2\n", + "Processing 1900 of 1000 on page 2\n", + "Processing 1901 of 1000 on page 2\n", + "Processing 1902 of 1000 on page 2\n", + "Processing 1903 of 1000 on page 2\n", + "Processing 1904 of 1000 on page 2\n", + "Processing 1905 of 1000 on page 2\n", + "Processing 1906 of 1000 on page 2\n", + "Processing 1907 of 1000 on page 2\n", + "Processing 1908 of 1000 on page 2\n", + "Processing 1909 of 1000 on page 2\n", + "Processing 1910 of 1000 on page 2\n", + "Processing 1911 of 1000 on page 2\n", + "Processing 1912 of 1000 on page 2\n", + "Processing 1913 of 1000 on page 2\n", + "Processing 1914 of 1000 on page 2\n", + "Processing 1915 of 1000 on page 2\n", + "Processing 1916 of 1000 on page 2\n", + "Processing 1917 of 1000 on page 2\n", + "Processing 1918 of 1000 on page 2\n", + "Processing 1919 of 1000 on page 2\n", + "Processing 1920 of 1000 on page 2\n", + "Processing 1921 of 1000 on page 2\n", + "Processing 1922 of 1000 on page 2\n", + "Processing 1923 of 1000 on page 2\n", + "Processing 1924 of 1000 on page 2\n", + "Processing 1925 of 1000 on page 2\n", + "Processing 1926 of 1000 on page 2\n", + "Processing 1927 of 1000 on page 2\n", + "Processing 1928 of 1000 on page 2\n", + "Processing 1929 of 1000 on page 2\n", + "Processing 1930 of 1000 on page 2\n", + "Processing 1931 of 1000 on page 2\n", + "Processing 1932 of 1000 on page 2\n", + "Processing 1933 of 1000 on page 2\n", + "Processing 1934 of 1000 on page 2\n", + "Processing 1935 of 1000 on page 2\n", + "Processing 1936 of 1000 on page 2\n", + "Processing 1937 of 1000 on page 2\n", + "Processing 1938 of 1000 on page 2\n", + "Processing 1939 of 1000 on page 2\n", + "Processing 1940 of 1000 on page 2\n", + "Processing 1941 of 1000 on page 2\n", + "Processing 1942 of 1000 on page 2\n", + "Processing 1943 of 1000 on page 2\n", + "Processing 1944 of 1000 on page 2\n", + "Processing 1945 of 1000 on page 2\n", + "Processing 1946 of 1000 on page 2\n", + "Processing 1947 of 1000 on page 2\n", + "Processing 1948 of 1000 on page 2\n", + "Processing 1949 of 1000 on page 2\n", + "Processing 1950 of 1000 on page 2\n", + "Processing 1951 of 1000 on page 2\n", + "Processing 1952 of 1000 on page 2\n", + "Processing 1953 of 1000 on page 2\n", + "Processing 1954 of 1000 on page 2\n", + "Processing 1955 of 1000 on page 2\n", + "Processing 1956 of 1000 on page 2\n", + "Processing 1957 of 1000 on page 2\n", + "Processing 1958 of 1000 on page 2\n", + "Processing 1959 of 1000 on page 2\n", + "Processing 1960 of 1000 on page 2\n", + "Processing 1961 of 1000 on page 2\n", + "Processing 1962 of 1000 on page 2\n", + "Processing 1963 of 1000 on page 2\n", + "Processing 1964 of 1000 on page 2\n", + "Processing 1965 of 1000 on page 2\n", + "Processing 1966 of 1000 on page 2\n", + "Processing 1967 of 1000 on page 2\n", + "Processing 1968 of 1000 on page 2\n", + "Processing 1969 of 1000 on page 2\n", + "Processing 1970 of 1000 on page 2\n", + "Processing 1971 of 1000 on page 2\n", + "Processing 1972 of 1000 on page 2\n", + "Processing 1973 of 1000 on page 2\n", + "Processing 1974 of 1000 on page 2\n", + "Processing 1975 of 1000 on page 2\n", + "Processing 1976 of 1000 on page 2\n", + "Processing 1977 of 1000 on page 2\n", + "Processing 1978 of 1000 on page 2\n", + "Processing 1979 of 1000 on page 2\n", + "Processing 1980 of 1000 on page 2\n", + "Processing 1981 of 1000 on page 2\n", + "Processing 1982 of 1000 on page 2\n", + "Processing 1983 of 1000 on page 2\n", + "Processing 1984 of 1000 on page 2\n", + "Processing 1985 of 1000 on page 2\n", + "Processing 1986 of 1000 on page 2\n", + "Processing 1987 of 1000 on page 2\n", + "Processing 1988 of 1000 on page 2\n", + "Processing 1989 of 1000 on page 2\n", + "Processing 1990 of 1000 on page 2\n", + "Processing 1991 of 1000 on page 2\n", + "Processing 1992 of 1000 on page 2\n", + "Processing 1993 of 1000 on page 2\n", + "Processing 1994 of 1000 on page 2\n", + "Processing 1995 of 1000 on page 2\n", + "Processing 1996 of 1000 on page 2\n", + "Processing 1997 of 1000 on page 2\n", + "Processing 1998 of 1000 on page 2\n", + "Processing 1999 of 1000 on page 2\n", + "Processing 2000 of 1000 on page 2\n", + "Processing 2001 of 153 on page 3\n", + "Processing 2002 of 153 on page 3\n", + "Processing 2003 of 153 on page 3\n", + "Processing 2004 of 153 on page 3\n", + "Processing 2005 of 153 on page 3\n", + "Processing 2006 of 153 on page 3\n", + "Processing 2007 of 153 on page 3\n", + "Processing 2008 of 153 on page 3\n", + "Processing 2009 of 153 on page 3\n", + "Processing 2010 of 153 on page 3\n", + "Processing 2011 of 153 on page 3\n", + "Processing 2012 of 153 on page 3\n", + "Processing 2013 of 153 on page 3\n", + "Processing 2014 of 153 on page 3\n", + "Processing 2015 of 153 on page 3\n", + "Processing 2016 of 153 on page 3\n", + "Processing 2017 of 153 on page 3\n", + "Processing 2018 of 153 on page 3\n", + "Processing 2019 of 153 on page 3\n", + "Processing 2020 of 153 on page 3\n", + "Processing 2021 of 153 on page 3\n", + "Processing 2022 of 153 on page 3\n", + "Processing 2023 of 153 on page 3\n", + "Processing 2024 of 153 on page 3\n", + "Processing 2025 of 153 on page 3\n", + "Processing 2026 of 153 on page 3\n", + "Processing 2027 of 153 on page 3\n", + "Processing 2028 of 153 on page 3\n", + "Processing 2029 of 153 on page 3\n", + "Processing 2030 of 153 on page 3\n", + "Processing 2031 of 153 on page 3\n", + "Processing 2032 of 153 on page 3\n", + "Processing 2033 of 153 on page 3\n", + "Processing 2034 of 153 on page 3\n", + "Processing 2035 of 153 on page 3\n", + "Processing 2036 of 153 on page 3\n", + "Processing 2037 of 153 on page 3\n", + "Processing 2038 of 153 on page 3\n", + "Processing 2039 of 153 on page 3\n", + "Processing 2040 of 153 on page 3\n", + "Processing 2041 of 153 on page 3\n", + "Processing 2042 of 153 on page 3\n", + "Processing 2043 of 153 on page 3\n", + "Processing 2044 of 153 on page 3\n", + "Processing 2045 of 153 on page 3\n", + "Processing 2046 of 153 on page 3\n", + "Processing 2047 of 153 on page 3\n", + "Processing 2048 of 153 on page 3\n", + "Processing 2049 of 153 on page 3\n", + "Processing 2050 of 153 on page 3\n", + "Processing 2051 of 153 on page 3\n", + "Processing 2052 of 153 on page 3\n", + "Processing 2053 of 153 on page 3\n", + "Processing 2054 of 153 on page 3\n", + "Processing 2055 of 153 on page 3\n", + "Processing 2056 of 153 on page 3\n", + "Processing 2057 of 153 on page 3\n", + "Processing 2058 of 153 on page 3\n", + "Processing 2059 of 153 on page 3\n", + "Processing 2060 of 153 on page 3\n", + "Processing 2061 of 153 on page 3\n", + "Processing 2062 of 153 on page 3\n", + "Processing 2063 of 153 on page 3\n", + "Processing 2064 of 153 on page 3\n", + "Processing 2065 of 153 on page 3\n", + "Processing 2066 of 153 on page 3\n", + "Processing 2067 of 153 on page 3\n", + "Processing 2068 of 153 on page 3\n", + "Processing 2069 of 153 on page 3\n", + "Processing 2070 of 153 on page 3\n", + "Processing 2071 of 153 on page 3\n", + "Processing 2072 of 153 on page 3\n", + "Processing 2073 of 153 on page 3\n", + "Processing 2074 of 153 on page 3\n", + "Processing 2075 of 153 on page 3\n", + "Processing 2076 of 153 on page 3\n", + "Processing 2077 of 153 on page 3\n", + "Processing 2078 of 153 on page 3\n", + "Processing 2079 of 153 on page 3\n", + "Processing 2080 of 153 on page 3\n", + "Processing 2081 of 153 on page 3\n", + "Processing 2082 of 153 on page 3\n", + "Processing 2083 of 153 on page 3\n", + "Processing 2084 of 153 on page 3\n", + "Processing 2085 of 153 on page 3\n", + "Processing 2086 of 153 on page 3\n", + "Processing 2087 of 153 on page 3\n", + "Processing 2088 of 153 on page 3\n", + "Processing 2089 of 153 on page 3\n", + "Processing 2090 of 153 on page 3\n", + "Processing 2091 of 153 on page 3\n", + "Processing 2092 of 153 on page 3\n", + "Processing 2093 of 153 on page 3\n", + "Processing 2094 of 153 on page 3\n", + "Processing 2095 of 153 on page 3\n", + "Processing 2096 of 153 on page 3\n", + "Processing 2097 of 153 on page 3\n", + "Processing 2098 of 153 on page 3\n", + "Processing 2099 of 153 on page 3\n", + "Processing 2100 of 153 on page 3\n", + "Processing 2101 of 153 on page 3\n", + "Processing 2102 of 153 on page 3\n", + "Processing 2103 of 153 on page 3\n", + "Processing 2104 of 153 on page 3\n", + "Processing 2105 of 153 on page 3\n", + "Processing 2106 of 153 on page 3\n", + "Processing 2107 of 153 on page 3\n", + "Processing 2108 of 153 on page 3\n", + "Processing 2109 of 153 on page 3\n", + "Processing 2110 of 153 on page 3\n", + "Processing 2111 of 153 on page 3\n", + "Processing 2112 of 153 on page 3\n", + "Processing 2113 of 153 on page 3\n", + "Processing 2114 of 153 on page 3\n", + "Processing 2115 of 153 on page 3\n", + "Processing 2116 of 153 on page 3\n", + "Processing 2117 of 153 on page 3\n", + "Processing 2118 of 153 on page 3\n", + "Processing 2119 of 153 on page 3\n", + "Processing 2120 of 153 on page 3\n", + "Processing 2121 of 153 on page 3\n", + "Processing 2122 of 153 on page 3\n", + "Processing 2123 of 153 on page 3\n", + "Processing 2124 of 153 on page 3\n", + "Processing 2125 of 153 on page 3\n", + "Processing 2126 of 153 on page 3\n", + "Processing 2127 of 153 on page 3\n", + "Processing 2128 of 153 on page 3\n", + "Processing 2129 of 153 on page 3\n", + "Processing 2130 of 153 on page 3\n", + "Processing 2131 of 153 on page 3\n", + "Processing 2132 of 153 on page 3\n", + "Processing 2133 of 153 on page 3\n", + "Processing 2134 of 153 on page 3\n", + "Processing 2135 of 153 on page 3\n", + "Processing 2136 of 153 on page 3\n", + "Processing 2137 of 153 on page 3\n", + "Processing 2138 of 153 on page 3\n", + "Processing 2139 of 153 on page 3\n", + "Processing 2140 of 153 on page 3\n", + "Processing 2141 of 153 on page 3\n", + "Processing 2142 of 153 on page 3\n", + "Processing 2143 of 153 on page 3\n", + "Processing 2144 of 153 on page 3\n", + "Processing 2145 of 153 on page 3\n", + "Processing 2146 of 153 on page 3\n", + "Processing 2147 of 153 on page 3\n", + "Processing 2148 of 153 on page 3\n", + "Processing 2149 of 153 on page 3\n", + "Processing 2150 of 153 on page 3\n", + "Processing 2151 of 153 on page 3\n", + "Processing 2152 of 153 on page 3\n", + "Processing 2153 of 153 on page 3\n", + "usgs_elev_tables load completed\n" + ] + } + ], "source": [ "\n", - "# NOTE: Aug 6, 2024 - There is already a 4.4.0.0 backup. How did it get made?\n", "# Has appx 2,150 HUCs to process, but this section goes quickly.\n", "\n", "sf.execute_sql('DROP TABLE IF EXISTS derived.usgs_elev_table;')\n", @@ -1132,10 +3217,11 @@ "page_iterator = paginator.paginate(**operation_parameters)\n", "page_count = 0\n", "for page in page_iterator:\n", - " page_count += 1\n", + " \n", " prefix_objects = page['CommonPrefixes']\n", " for i, prefix_obj in enumerate(prefix_objects):\n", - " print(f\"Processing {i+1} of {len(prefix_objects)} on page {page_count}\")\n", + " print(f\"Processing {(i+1) + (1000 * page_count)} of\"\n", + " f\"{len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\")\n", " huc_prefix = prefix_obj.get(\"Prefix\")\n", " usgs_elev_table_key = f'{huc_prefix}usgs_elev_table.csv'\n", " try:\n", @@ -1162,6 +3248,9 @@ " pass\n", " else:\n", " raise e\n", + "\n", + " page_count += 1\n", + " \n", " \n", "print(\"usgs_elev_tables load completed\")\n" ] @@ -1174,34 +3263,11 @@ "

8 - Recreate derived.hydrotable_staggered

" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "18a2a7c2-7499-4ebc-a878-769e62dfbd6b", - "metadata": { - "jupyter": { - "source_hidden": true - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# Already run for 4.4.0.0 (4.5.2.11)\n", - "\n", - "# TODO: Aug 2024: Change this to a backup without indexes and not rename\n", - "\n", - "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.hydrotable_staggered RENAME TO hydrotable_staggered_{OLD_FIM_TAG};')\n", - "#print(\"derived.hydrotable_staggered renamed\")" - ] - }, { "cell_type": "code", "execution_count": null, "id": "33026eb6-fbc7-47f7-a44b-f9cea8f0911b", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1226,10 +3292,11 @@ "page_iterator = paginator.paginate(**operation_parameters)\n", "page_count = 0\n", "for page in page_iterator:\n", - " page_count += 1\n", + "\n", " prefix_objects = page['CommonPrefixes']\n", " for i, prefix_obj in enumerate(prefix_objects):\n", - " print(f\"Processing {i+1} of {len(prefix_objects)} on page {page_count}\")\n", + " print(f\"Processing {(i+1) + (1000 * page_count)} of\"\n", + " f\" {len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\")\n", " branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", " branch_files_result = S3_CLIENT.list_objects(\n", " Bucket=FIM_BUCKET, \n", @@ -1266,6 +3333,9 @@ " raise e\n", " print(f'Fetch failed: {e}')\n", " \n", + " page_count += 1\n", + " \n", + " \n", "end_dt = datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(\"hydrotable reload done\")\n", @@ -1277,9 +3347,6 @@ "execution_count": null, "id": "5cc764e4-7e67-4b51-a100-6021e31416cd", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1316,9 +3383,6 @@ "execution_count": null, "id": "74f21c03-ca36-402c-a69b-396162720f66", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1344,9 +3408,6 @@ "execution_count": null, "id": "461f20b4-8442-4c0e-9114-ea62764cd789", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1367,9 +3428,6 @@ "execution_count": null, "id": "b752fce5-c8d7-4064-92d7-842c22d1723e", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1389,9 +3447,6 @@ "execution_count": null, "id": "d6a0cc18-de82-4bd8-bd4d-c3ad021a1dec", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1422,9 +3477,6 @@ "execution_count": null, "id": "4548d5e8-303e-44e0-a8a5-7b1947214b4e", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1456,9 +3508,6 @@ "execution_count": null, "id": "d1a77875-f569-4410-888c-b132714a7c9e", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1484,9 +3533,6 @@ "execution_count": null, "id": "95c472f8-0557-4864-8536-1814f3ac4286", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1530,9 +3576,6 @@ "execution_count": null, "id": "9a249854-b9cd-4ccb-afea-96c35708515d", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1548,9 +3591,6 @@ "execution_count": null, "id": "c24f509a-90df-492a-a0d4-aa9eab8e7041", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1582,9 +3622,6 @@ "execution_count": null, "id": "b5639447-9a5a-4add-9c36-ebd945a5a8d0", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1638,9 +3675,6 @@ "execution_count": null, "id": "0df23345-47a4-490a-b10f-2020e3a2a68e", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1665,9 +3699,6 @@ "execution_count": null, "id": "b291a461-e36f-4049-84b6-dd59cd1c4ad2", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1715,9 +3746,6 @@ "execution_count": null, "id": "f4fcdfab-3936-4a0a-8ab9-605e9a64f9bd", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1747,9 +3775,6 @@ "execution_count": null, "id": "06ca5581", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1971,31 +3996,9 @@ " 'geometry': 'geom',\n", " 'huc': 'huc8'})\n", "\n", - " # 4.5.2.11, fixing a column name bug\n", - " if catfim_type == 'stage_based_catfim_sites':\n", - " df = df.rename(columns={'nws_lid': 'ahps_lid'})\n", - "\n", " # Convert all field names to lowercase (needed for ArcGIS Pro).\n", " df.columns = df.columns.str.lower()\n", "\n", - " # Remove sites that are in derived.ahps_restricted_sites\n", - " # TODO: Aug 2024: Need to see if this list needs to be updated. Submitted card.\n", - " restricted_sites_df = sf.get_db_values(\"derived.ahps_restricted_sites\", [\"*\"])\n", - " restricted_dict = restricted_sites_df.to_dict('records')\n", - "\n", - " for site in restricted_dict:\n", - " nws_lid = site['nws_lid'].lower()\n", - " #print(nws_lid)\n", - " if \"sites\" in catfim_type:\n", - " # print(True)\n", - " # print(nws_lid)\n", - " df.loc[df.ahps_lid == nws_lid, 'mapped'] = 'no'\n", - " df.loc[df.ahps_lid == nws_lid, 'status'] = site['restricted_reason']\n", - " # print(df.loc[df.ahps_lid==nws_lid]['status'])\n", - " else:\n", - " df.loc[df.ahps_lid == nws_lid, 'viz'] = 'no'\n", - " df = df[df['viz'] == 'yes']\n", - "\n", " # Enforce data types on df before loading in DB (TODO: need to create special cases for each layer).\n", " df = df.astype({'huc8': 'str'})\n", " df = df.fillna(0)\n", @@ -2023,9 +4026,9 @@ " df = df.astype({'nws_data_rfc_defined_fcst_point': 'str'})\n", " df = df.astype({'nws_data_riverpoint': 'str'})\n", "\n", - " # TODO: Aug 27, 2024: For now, let's jsut override the \"version\" column and fix it when we\n", - " # reconsile the fim_version and model_version columns\n", - " df['version'] = PUBLIC_FIM_VERSION\n", + "\n", + " # As of Nov 1, 2024: Ignore the incoming \"version\" from dataset\n", + " # df['version'] = PUBLIC_FIM_VERSION\n", " df[COLUMN_NAME_FIM_VERSION] = PUBLIC_FIM_VERSION\n", " df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", "\n", @@ -2110,9 +4113,6 @@ "execution_count": null, "id": "f9577edb-4aa6-423b-819e-df8c922c7ec2", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -2221,9 +4221,6 @@ "execution_count": null, "id": "f5cf108d-7360-48a9-a4a2-81b23b9e51b4", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -2276,9 +4273,6 @@ "execution_count": null, "id": "fd38c00d-22ad-476e-ab05-cf293e5bbc15", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -2338,9 +4332,6 @@ "execution_count": null, "id": "a3d1ab20-4148-4d1d-bbb6-a2c3983ba145", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], diff --git a/Core/Manual_Workflows/__init__.py b/Core/Manual_Workflows/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Core/__init__.py b/Core/__init__.py new file mode 100644 index 00000000..e69de29b From 19eadacec2f6dd386a43f75e96f0791fb73dabd9 Mon Sep 17 00:00:00 2001 From: RobH Date: Sun, 10 Nov 2024 18:28:54 +0000 Subject: [PATCH 08/18] wip --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 66 +++++++++++++++---- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 26251c4b..6faac13b 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -112,9 +112,6 @@ "execution_count": 2, "id": "5b656259", "metadata": { - "jupyter": { - "source_hidden": true - }, "scrolled": true, "tags": [] }, @@ -209,7 +206,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 17, "id": "e944eff3-6023-48f4-9f57-27304a240447", "metadata": { "tags": [] @@ -225,11 +222,13 @@ ], "source": [ "\n", - "PUBLIC_FIM_VERSION = \"FIM 5.1.0\"\n", - "FIM_MODEL_VERSION = \"HAND 4.5.11.1\"\n", - "RAS2FIM_MODEL_VERSION = \"ras2fim 2.0.1.0\"\n", + "# we won't load this into any tables at this time\n", + "# The phrase of FIM 5.1.0 will be embedded in config files\n", + "#PU0LIC_FIM_VERSION = \"FIM 5.1.0\"\n", + "HAND_MODEL_VERSION = \"4.5.11.1\"\n", + "RAS2FIM_MODEL_VERSION = \"2.0\"\n", "\n", - "FIM_ROOT_DPATH = \"fim/hand_4_5_11_1\"\n", + "HAND_ROOT_DPATH = \"fim/hand_4_5_11_1\"\n", "HAND_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/hand_datasets\"\n", "QA_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/qa_datasets\"\n", "\n", @@ -237,7 +236,6 @@ "FIM_CROSSWALK_FPATH = os.path.join(HAND_DATASETS_DPATH, \"crosswalk_table.csv\")\n", "PIPELINE_ARN = 'arn:aws:states:us-east-1:526904826677:stateMachine:hv-vpp-ti-viz-pipeline'\n", "\n", - "COLUMN_NAME_FIM_VERSION = \"fim_version\"\n", "COLUMN_NAME_MODEL_VERSION = \"model_version\"\n", "\n", "# Sometimes these credential values get updated. To find the latest correct values, go to your AWS Console log page and click on the \"Access Key\"\n", @@ -268,9 +266,6 @@ "execution_count": null, "id": "c96a49f2", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -3265,12 +3260,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "33026eb6-fbc7-47f7-a44b-f9cea8f0911b", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hydrotable reloaded - started\n", + "Processing 1 of 0 on page 1 (1000 per page)\n", + "Processing 1002 of 1000 on page 2 (1000 per page)\n", + "Processing 2003 of 2000 on page 3 (1000 per page)\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[16], line 46\u001b[0m\n\u001b[1;32m 41\u001b[0m ht \u001b[38;5;241m=\u001b[39m S3_CLIENT\u001b[38;5;241m.\u001b[39mget_object(\n\u001b[1;32m 42\u001b[0m Bucket\u001b[38;5;241m=\u001b[39mFIM_BUCKET,\n\u001b[1;32m 43\u001b[0m Key\u001b[38;5;241m=\u001b[39mhydro_table_key\n\u001b[1;32m 44\u001b[0m )[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mBody\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# print(\"...Reading with pandas...\")\u001b[39;00m\n\u001b[0;32m---> 46\u001b[0m ht_df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mht\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheader\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43musecols\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mht_usecols\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;66;03m# print('...Writing to db...')\u001b[39;00m\n\u001b[1;32m 48\u001b[0m ht_df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfim_version\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m PUBLIC_FIM_VERSION\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1026\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m 1013\u001b[0m kwds_defaults \u001b[38;5;241m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m 1014\u001b[0m dialect,\n\u001b[1;32m 1015\u001b[0m delimiter,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1022\u001b[0m dtype_backend\u001b[38;5;241m=\u001b[39mdtype_backend,\n\u001b[1;32m 1023\u001b[0m )\n\u001b[1;32m 1024\u001b[0m kwds\u001b[38;5;241m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m-> 1026\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:626\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 623\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parser\n\u001b[1;32m 625\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m parser:\n\u001b[0;32m--> 626\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mparser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1923\u001b[0m, in \u001b[0;36mTextFileReader.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 1916\u001b[0m nrows \u001b[38;5;241m=\u001b[39m validate_integer(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnrows\u001b[39m\u001b[38;5;124m\"\u001b[39m, nrows)\n\u001b[1;32m 1917\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1918\u001b[0m \u001b[38;5;66;03m# error: \"ParserBase\" has no attribute \"read\"\u001b[39;00m\n\u001b[1;32m 1919\u001b[0m (\n\u001b[1;32m 1920\u001b[0m index,\n\u001b[1;32m 1921\u001b[0m columns,\n\u001b[1;32m 1922\u001b[0m col_dict,\n\u001b[0;32m-> 1923\u001b[0m ) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# type: ignore[attr-defined]\u001b[39;49;00m\n\u001b[1;32m 1924\u001b[0m \u001b[43m \u001b[49m\u001b[43mnrows\u001b[49m\n\u001b[1;32m 1925\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1926\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 1927\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/io/parsers/c_parser_wrapper.py:234\u001b[0m, in \u001b[0;36mCParserWrapper.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 233\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlow_memory:\n\u001b[0;32m--> 234\u001b[0m chunks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_low_memory\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;66;03m# destructive to chunks\u001b[39;00m\n\u001b[1;32m 236\u001b[0m data \u001b[38;5;241m=\u001b[39m _concatenate_chunks(chunks)\n", + "File \u001b[0;32mparsers.pyx:838\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader.read_low_memory\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32mparsers.pyx:905\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32mparsers.pyx:874\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._tokenize_rows\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32mparsers.pyx:891\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._check_tokenize_status\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32mparsers.pyx:2053\u001b[0m, in \u001b[0;36mpandas._libs.parsers.raise_parser_error\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/response.py:99\u001b[0m, in \u001b[0;36mStreamingBody.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Read at most amt bytes from the stream.\u001b[39;00m\n\u001b[1;32m 95\u001b[0m \n\u001b[1;32m 96\u001b[0m \u001b[38;5;124;03mIf the amt argument is omitted, read all data.\u001b[39;00m\n\u001b[1;32m 97\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 99\u001b[0m chunk \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raw_stream\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 100\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m URLLib3ReadTimeoutError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 101\u001b[0m \u001b[38;5;66;03m# TODO: the url will be None as urllib3 isn't setting it yet\u001b[39;00m\n\u001b[1;32m 102\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ReadTimeoutError(endpoint_url\u001b[38;5;241m=\u001b[39me\u001b[38;5;241m.\u001b[39murl, error\u001b[38;5;241m=\u001b[39me)\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/urllib3/response.py:949\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt, decode_content, cache_content)\u001b[0m\n\u001b[1;32m 946\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m amt:\n\u001b[1;32m 947\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer\u001b[38;5;241m.\u001b[39mget(amt)\n\u001b[0;32m--> 949\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raw_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 951\u001b[0m flush_decoder \u001b[38;5;241m=\u001b[39m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (amt \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data)\n\u001b[1;32m 953\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/urllib3/response.py:873\u001b[0m, in \u001b[0;36mHTTPResponse._raw_read\u001b[0;34m(self, amt, read1)\u001b[0m\n\u001b[1;32m 870\u001b[0m fp_closed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclosed\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 872\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_error_catcher():\n\u001b[0;32m--> 873\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mread1\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mread1\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m fp_closed \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 874\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[1;32m 875\u001b[0m \u001b[38;5;66;03m# Platform-specific: Buggy versions of Python.\u001b[39;00m\n\u001b[1;32m 876\u001b[0m \u001b[38;5;66;03m# Close the connection when no data is returned\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 881\u001b[0m \u001b[38;5;66;03m# not properly close the connection in all cases. There is\u001b[39;00m\n\u001b[1;32m 882\u001b[0m \u001b[38;5;66;03m# no harm in redundantly calling close.\u001b[39;00m\n\u001b[1;32m 883\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mclose()\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/urllib3/response.py:856\u001b[0m, in \u001b[0;36mHTTPResponse._fp_read\u001b[0;34m(self, amt, read1)\u001b[0m\n\u001b[1;32m 853\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread1(amt) \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread1()\n\u001b[1;32m 854\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 855\u001b[0m \u001b[38;5;66;03m# StringIO doesn't like amt=None\u001b[39;00m\n\u001b[0;32m--> 856\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread()\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/http/client.py:466\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength:\n\u001b[1;32m 464\u001b[0m \u001b[38;5;66;03m# clip the read to the \"end of response\"\u001b[39;00m\n\u001b[1;32m 465\u001b[0m amt \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength\n\u001b[0;32m--> 466\u001b[0m s \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 467\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m s \u001b[38;5;129;01mand\u001b[39;00m amt:\n\u001b[1;32m 468\u001b[0m \u001b[38;5;66;03m# Ideally, we would raise IncompleteRead if the content-length\u001b[39;00m\n\u001b[1;32m 469\u001b[0m \u001b[38;5;66;03m# wasn't satisfied, but it might break compatibility.\u001b[39;00m\n\u001b[1;32m 470\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_conn()\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/socket.py:705\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m 704\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 705\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 706\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m 707\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/ssl.py:1307\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m 1303\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 1304\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 1305\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m 1306\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1307\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1308\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1309\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n", + "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/ssl.py:1163\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m 1161\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1162\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1163\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1164\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], "source": [ "\n", "# Takes appx 5.75 to 6 hrs to run\n", @@ -3295,6 +3329,10 @@ "\n", " prefix_objects = page['CommonPrefixes']\n", " for i, prefix_obj in enumerate(prefix_objects):\n", + " \n", + " print(f\"Processing {(i+1) + (1000 * page_count)} of\"\n", + " f\"{len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\") \n", + " \n", " print(f\"Processing {(i+1) + (1000 * page_count)} of\"\n", " f\" {len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\")\n", " branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", From e85e8e067517e548981f52ae8391ae5b4f12cf98 Mon Sep 17 00:00:00 2001 From: RobH Date: Wed, 13 Nov 2024 22:46:13 +0000 Subject: [PATCH 09/18] WIP: loading data --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 240 ++++++++++++++---- 1 file changed, 188 insertions(+), 52 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 6faac13b..444c0ce4 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -226,7 +226,6 @@ "# The phrase of FIM 5.1.0 will be embedded in config files\n", "#PU0LIC_FIM_VERSION = \"FIM 5.1.0\"\n", "HAND_MODEL_VERSION = \"4.5.11.1\"\n", - "RAS2FIM_MODEL_VERSION = \"2.0\"\n", "\n", "HAND_ROOT_DPATH = \"fim/hand_4_5_11_1\"\n", "HAND_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/hand_datasets\"\n", @@ -266,6 +265,9 @@ "execution_count": null, "id": "c96a49f2", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -318,15 +320,10 @@ " \"\"\"\n", " cur.execute(sql)\n", " conn.commit()\n", - " \n", - " \n", - " print(f\"Adding {COLUMN_NAME_FIM_VERSION} column to derived.fim4_featureid_crosswalk\")\n", - " sql = f\"ALTER TABLE derived.fim4_featureid_crosswalk ADD COLUMN IF NOT EXISTS {COLUMN_NAME_FIM_VERSION} text DEFAULT '{PUBLIC_FIM_VERSION}';\"\n", - " cur.execute(sql)\n", - " conn.commit()\n", + "\n", " \n", " print(f\"Adding {COLUMN_NAME_MODEL_VERSION} column to derived.fim4_featureid_crosswalk\")\n", - " sql = f\"ALTER TABLE derived.fim4_featureid_crosswalk ADD COLUMN IF NOT EXISTS {COLUMN_NAME_MODEL_VERSION} text DEFAULT '{FIM_MODEL_VERSION}';\"\n", + " sql = f\"ALTER TABLE derived.fim4_featureid_crosswalk ADD COLUMN IF NOT EXISTS {COLUMN_NAME_MODEL_VERSION} text DEFAULT '{HAND_MODEL_VERSION}';\"\n", " cur.execute(sql)\n", " conn.commit()\n", " \n", @@ -353,6 +350,43 @@ "print(\"... Estimated time to completion is just a few mins\")\n" ] }, + { + "cell_type": "code", + "execution_count": 18, + "id": "28a5c72d-e96a-45da-b4a3-a57d04e23013", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "***> Established db connection to: hv-vpp-ti-viz-processing.c4vzypepnkx3.us-east-1.rds.amazonaws.com from ()\n", + "table updated\n" + ] + } + ], + "source": [ + "# Nov 10, 2024\n", + "# After initial upload, we decided to no longer have a fim_version field in any table, just drop it\n", + "# Also.. change the model_version tables to be purely dot notation numbers. ie) 4.5.11.1\n", + "\n", + "db = database(db_type=\"viz\")\n", + "with db.get_db_connection() as conn, conn.cursor() as cur:\n", + " \n", + " sql = \"ALTER TABLE derived.fim4_featureid_crosswalk DROP COLUMN fim_version\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + " \n", + " sql = \"UPDATE derived.fim4_featureid_crosswalk SET model_version = '4.5.11.1'\"\n", + " cur.execute(sql)\n", + " conn.commit()\n", + "\n", + "\n", + "print(\"table updated\")" + ] + }, { "cell_type": "markdown", "id": "c17fdbfc-6f88-483d-a76d-29ebd525c5e5", @@ -373,7 +407,7 @@ }, { "cell_type": "markdown", - "id": "13507d85-ff04-42f3-bf72-b896d63d4679", + "id": "c2565d24-2dc5-4240-ae37-fa5374a660d4", "metadata": { "tags": [] }, @@ -396,15 +430,14 @@ "
\n", "\n", "#### Note: Later in these steps we will change the Memory and Emphermal Storage back to default values, see below ####\n", - "Nov 5, 2024: Added new variable called \"MODEL_VERSION\" : \"HAND 4.5.11.1\". FIM_VERSION IS NOW: 5.1.0\n", - "\n", + "Nov 5, 2024: Added new variable called \"MODEL_VERSION\" : \"4.5.11.1\". --- FIM_VERSION IS NOW: (blank). field might be removed later)\n", "\n", "\n" ] }, { "cell_type": "markdown", - "id": "9f19f7ac-61d1-4323-8138-6823da658aa3", + "id": "052f15eb-c883-4c27-9dd0-eb9c77ae7594", "metadata": { "tags": [] }, @@ -417,17 +450,16 @@ "***The code for ras2fim is removed here from the 4.5.2.11 set and will rebuilt as it''s own new separate load script when that happens.***\n", "\n", "However, we will have a few modifications for ras2fim data (not a reload) to help bring in the new\n", - "fim_version and model_version columns. Those changes are included here.\n" + "fim_version and model_version format of just \"2.0\" columns. Those changes are included here.\n", + "\n", + "The Fim Version field needs to be deleted\n" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 23, "id": "dfc0a44c-8b46-4db5-a9a0-c65e0bf1e20b", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [ @@ -435,7 +467,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Updated geocurves table to new FIM version value\n" + "Updating geocurves table to model_version of 2.0 to follow the new versioning system and drop fim_version\n", + "Updating done for geocurves\n" ] } ], @@ -443,14 +476,12 @@ "\n", "# Update \"geocurves\" to update the \"fim_version\" field to \"FIM 5.1.0:\n", "\n", - "print(f\"Updating geocurves table to fim_version of {PUBLIC_FIM_VERSION}\"\n", + "print(\"Updating geocurves table to model_version of 2.0 to follow the new versioning system and drop fim_version\")\n", + "\n", + "sf.execute_sql(\"UPDATE ras2fim.geocurves SET model_version = '2.0'\", db_type=\"viz\")\n", "\n", - "sf.execute_sql(f'''\n", - "UPDATE\n", - " ras2fim.geocurves\n", - "SET\n", - " fim_version = '{PUBLIC_FIM_VERSION}';\n", - "''', db_type=\"viz\")\n", + "# Drop the \"fim_version\" field.\n", + "sf.execute_sql(\"ALTER TABLE ras2fim.geocurves DROP COLUMN fim_version\", db_type=\"viz\")\n", "\n", "print(\"Updating done for geocurves\")\n", "\n" @@ -458,7 +489,7 @@ }, { "cell_type": "markdown", - "id": "0c5994fa-8fa5-471d-86e9-745d67e79ecf", + "id": "4c6d68b8-ecf2-406c-aa5d-43fe53707862", "metadata": { "tags": [] }, @@ -482,17 +513,29 @@ "A couple other important notes:\n", "- These AEP configurations write data directly to the aep_fim schema in the egis RDS database, instead of the viz database.\n", "- You'll need to dump the aep_fim schema after that is complete for backup / deployment into other environments.\n", - "- This process has not been tested with new NWM 3.0 Recurrence Flows, and a good thorough audit / QC check of output data is warranted, given those changes and the recent updates to the pipelines.\n" + "- This process has not been tested with new NWM 3.0 Recurrence Flows, and a good thorough audit / QC check of output data is warranted, given those changes and the recent updates to the pipelines.\n", + "\n", + "***Note: You can start each of these 6 one right after the other. Maybe someday we can create a block that just starts all 6 at once.***\n", + "\n", + "\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "6a698067", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "function: get_aep_pipeline_input loaded\n" + ] + } + ], "source": [ "\n", "# Aug 6, 2024: Note: This was created after all intervals were created, so only HW was tested against\n", @@ -548,12 +591,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "f6d6ee69", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 45 mins. Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_2_20241111T1720\n" + ] + } + ], "source": [ "\n", "#### 2 Year Flow\n", @@ -568,17 +619,27 @@ " input= json.dumps(pipeline_input)\n", ")\n", "\n", - "print(f\"AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + "print(\"AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 15 - 45 mins.\"\n", + "print(f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: {pipeline_name}\")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "a4f89d9a", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AEP : 5 year flows ie: rf_5_inundation kicked off. Can take 15 - 45 mins.\n", + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_5_20241111T1743\n" + ] + } + ], "source": [ "\n", "#### 5 Year Flow\n", @@ -593,17 +654,31 @@ " input= json.dumps(pipeline_input)\n", ")\n", "\n", - "print(f\"AEP : 5 year flows ie: rf_5_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + "print(f\"AEP : 5 year flows ie: rf_5_inundation kicked off. Can take 15 - 45 mins.\")\n", + "print(f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: {pipeline_name}\")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "791d1a8b", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AEP : 10 year flows ie: rf_10_inundation kicked off. Can take 15 - 45 mins.\n", + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_10_20241111T1743\n" + ] + } + ], "source": [ "\n", "#### 10 Year Flow\n", @@ -618,17 +693,31 @@ " input= json.dumps(pipeline_input)\n", ")\n", "\n", - "print(f\"AEP : 10 year flows ie: rf_10_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + "print(f\"AEP : 10 year flows ie: rf_10_inundation kicked off. Can take 15 - 45 mins.\")\n", + "print(f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: {pipeline_name}\")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "1bb87128", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AEP : 25 year flows ie: rf_25_inundation kicked off. Can take 15 - 45 mins.\n", + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_25_20241111T1743\n" + ] + } + ], "source": [ "\n", "#### 25 Year Flow\n", @@ -643,17 +732,31 @@ " input= json.dumps(pipeline_input)\n", ")\n", "\n", - "print(f\"AEP : 25 year flows ie: rf_25_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + "print(f\"AEP : 25 year flows ie: rf_25_inundation kicked off. Can take 15 - 45 mins.\")\n", + "print(f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: {pipeline_name}\")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "4832e4e0", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AEP : 50 year flows ie: rf_50_inundation kicked off. Can take 15 - 45 mins.\n", + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_50_20241111T1743\n" + ] + } + ], "source": [ "\n", "#### 50 Year Flow\n", @@ -668,17 +771,32 @@ " input= json.dumps(pipeline_input)\n", ")\n", "\n", - "print(f\"AEP : 50 year flows ie: rf_50_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + "print(f\"AEP : 50 year flows ie: rf_50_inundation kicked off. Can take 15 - 45 mins.\")\n", + "print(f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: {pipeline_name}\")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "187e83bc-ebbe-4615-a046-e0ef7b09ad3b", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AEP : High Water year flows ie: rf_hw_inundation kicked off. Can take 15 - 45 mins.\n", + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_hw_20241111T1743\n", + "\n" + ] + } + ], "source": [ "\n", "#### HW (High Water) Flow\n", @@ -693,7 +811,8 @@ " input= json.dumps(pipeline_input)\n", ")\n", "\n", - "print(f\"AEP : High Water year flows ie: rf_hw_inundation kicked off. Can take 45 mins. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n", + "print(f\"AEP : High Water year flows ie: rf_hw_inundation kicked off. Can take 15 - 45 mins.\")\n", + "print(f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: {pipeline_name}\")\n", "print(\"\")" ] }, @@ -731,12 +850,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "c77d6e5b-f5fb-422f-a6c3-0f53743cd631", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Catchment Truncation for Branch 0 Done\n", + "\n" + ] + } + ], "source": [ "# TODO: Add backups to these (4.4.0.0) (already not available for 4.4.0.0)\n", "\n", @@ -757,12 +885,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "id": "a7495759", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Catchments Branch 0 load kicked off. Last runtime: 23:38.019. Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - sagemaker_0_catchments_20241111T1815\n" + ] + } + ], "source": [ "pipeline_input = {\n", " \"configuration\": \"reference\",\n", @@ -866,7 +1002,7 @@ ")\n", "\n", "print(\"Catchments Branch 0 load kicked off. Last runtime: 23:38.019. \"\n", - " f\"Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + " f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - {pipeline_name}\")\n" ] }, { From 112afe487d46c168551e296e33ca7cb0b98ddbbb Mon Sep 17 00:00:00 2001 From: RobH Date: Sun, 17 Nov 2024 17:38:27 +0000 Subject: [PATCH 10/18] merging latest ti in --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 2525 +---------------- 1 file changed, 117 insertions(+), 2408 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 444c0ce4..87a16d2b 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -16,54 +16,11 @@ "WIP until you see a load date below.\n" ] }, - { - "cell_type": "markdown", - "id": "ed4c0734-01e8-4f37-a718-402d9be7a076", - "metadata": { - "tags": [] - }, - "source": [ - "### Load Status for hand 4.5.11.1 - Started Oct 31, 2024 (well... restarted from the 21st)\n", - "\n", - "#### Add dates to each line as they have been loaded\n", - "\n", - "1. `Crosswalk` : --- Done: Nov 5\n", - "2. `Lambda FIM_PREFIX` : --- Done: Nov 5\n", - "3. `Lambda FIM_VERSION and Memory` : --- Done: Nov 5\n", - "4. `ras2fim` : No update in this release. But a few adjustments for fim_version and model_version here : -- Done: Nov 5\n", - "5. `AEP` --- (HOLD) - needs lambda and image updates (via tf)\n", - " - `2 year` : -- \n", - " - `5 year` : -- \n", - " - `10 year` : -- \n", - " - `25 year` : -- \n", - " - `50 year` : -- \n", - " - `HW / High Water` : -- \n", - " - `Change the hv-vpp-ti-viz-fim-data-prep Lambda memory back to 2048mb` : -- \n", - "6. `Catchments` --- (HOLD) - needs lambda and image updates (via tf)\n", - " - `Branch 0` : -- \n", - " - `GMS` : -- \n", - "7. `usgs_elev_table` : -- Done Nov 7\n", - "8. `hydrotable / hydrotable_staggered` : -- \n", - "9. `usgs_rating_curve / usgs_rating_curves staggered` : -- \n", - "10. `Skills Metrics` : -- \n", - "11. `FIM Performance` : -- \n", - "12. `CatFIM`\n", - " - `Stage Based CatFIM` : -- \n", - " - `Flow Based CatFIM` : -- \n", - " - `CatFIM FIM 30` : Stage based only? flow not needed but confirm this.\n", - "13. `Clear HAND cache` :\n", - "14. `GIT` and `Terraform ??` : We can now do GitHub check in from here. Watch for branches.\n", - "\n" - ] - }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 1, "id": "9e1ee2b5-b109-49e8-8255-f06449b44ee6", "metadata": { - "jupyter": { - "source_hidden": true - }, "scrolled": true, "tags": [] }, @@ -72,11 +29,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Collecting python-dotenv\n", - " Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n", - "Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", - "Installing collected packages: python-dotenv\n", - "Successfully installed python-dotenv-1.0.1\n", + "Requirement already satisfied: python-dotenv in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.0.1)\n", "All loaded\n" ] } @@ -163,7 +116,7 @@ "from sqlalchemy.types import Text # yes, reduntant, fix it later\n", "\n", "\n", - "sys.path.append(os.path.join(os.path.abspath(''), '..'))\n", + "sys.path.append(os.path.abspath('..'))\n", "\n", "import helper_functions.shared_functions as sf\n", "import helper_functions.s3_shared_functions as s3_sf\n", @@ -175,15 +128,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "fae5ca0f-e582-4ff2-89b6-087285433a97", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "aws_keys are at /home/ec2-user/SageMaker/AWS_keys.env\n", + "aws_keys loaded\n" + ] + } + ], "source": [ "# Load AWS Keys\n", "from dotenv import load_dotenv\n", @@ -206,7 +165,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 4, "id": "e944eff3-6023-48f4-9f57-27304a240447", "metadata": { "tags": [] @@ -228,8 +187,8 @@ "HAND_MODEL_VERSION = \"4.5.11.1\"\n", "\n", "HAND_ROOT_DPATH = \"fim/hand_4_5_11_1\"\n", - "HAND_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/hand_datasets\"\n", - "QA_DATASETS_DPATH = f\"{FIM_ROOT_DPATH}/qa_datasets\"\n", + "HAND_DATASETS_DPATH = f\"{HAND_ROOT_DPATH}/hand_datasets\"\n", + "QA_DATASETS_DPATH = f\"{HAND_ROOT_DPATH}/qa_datasets\"\n", "\n", "FIM_BUCKET = \"hydrovis-ti-deployment-us-east-1\"\n", "FIM_CROSSWALK_FPATH = os.path.join(HAND_DATASETS_DPATH, \"crosswalk_table.csv\")\n", @@ -262,15 +221,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "c96a49f2", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Getting column name from fim/hand_4_5_11_1/hand_datasets/crosswalk_table.csv\n", + "(hand_id integer,feature_id bigint,huc8 TEXT,branch_id bigint,hydro_id integer,lake_id integer)\n", + "***> Established db connection to: hv-vpp-ti-viz-processing.c4vzypepnkx3.us-east-1.rds.amazonaws.com from ()\n", + "Deleting/Creating derived.fim4_featureid_crosswalk using columns (hand_id integer,feature_id bigint,huc8 TEXT,branch_id bigint,hydro_id integer,lake_id integer)\n", + "Importing fim/hand_4_5_11_1/hand_datasets/crosswalk_table.csv to derived.fim4_featureid_crosswalk\n", + "Adding model_version column to derived.fim4_featureid_crosswalk\n", + "Adding feature id index to derived.fim4_featureid_crosswalk\n", + "Adding hydro id index to derived.fim4_featureid_crosswalk\n", + "\n", + "... Estimated time to completion is just a few mins\n", + "Successully loaded derived.fim4_featureid_crosswalk and updated it\n" + ] + } + ], "source": [ "\n", "print(f\"Getting column name from {FIM_CROSSWALK_FPATH}\")\n", @@ -346,50 +320,13 @@ " conn.commit()\n", "\n", "print(\"\")\n", - "print(\"Successully loaded derived.fim4_featureid_crosswalk and updated it\")\n", - "print(\"... Estimated time to completion is just a few mins\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "28a5c72d-e96a-45da-b4a3-a57d04e23013", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "***> Established db connection to: hv-vpp-ti-viz-processing.c4vzypepnkx3.us-east-1.rds.amazonaws.com from ()\n", - "table updated\n" - ] - } - ], - "source": [ - "# Nov 10, 2024\n", - "# After initial upload, we decided to no longer have a fim_version field in any table, just drop it\n", - "# Also.. change the model_version tables to be purely dot notation numbers. ie) 4.5.11.1\n", - "\n", - "db = database(db_type=\"viz\")\n", - "with db.get_db_connection() as conn, conn.cursor() as cur:\n", - " \n", - " sql = \"ALTER TABLE derived.fim4_featureid_crosswalk DROP COLUMN fim_version\"\n", - " cur.execute(sql)\n", - " conn.commit()\n", - " \n", - " sql = \"UPDATE derived.fim4_featureid_crosswalk SET model_version = '4.5.11.1'\"\n", - " cur.execute(sql)\n", - " conn.commit()\n", - "\n", - "\n", - "print(\"table updated\")" + "print(\"... Estimated time to completion is just a few mins\")\n", + "print(\"Successully loaded derived.fim4_featureid_crosswalk and updated it\")\n" ] }, { "cell_type": "markdown", - "id": "c17fdbfc-6f88-483d-a76d-29ebd525c5e5", + "id": "82c7d50b-0885-4bec-8094-7efe3d762da4", "metadata": { "tags": [] }, @@ -400,14 +337,14 @@ "\n", "Lambda name: hv-vpp-ti-viz-hand-fim-processing\n", "\n", - "In the Configuration Tab, click on the `Environment variables` (left menu), then change the `FIX_PREFIX` to location of the latest hand_dataset you are working on. Referencial to S3 Bucket name.\n", - "
\n", - "ie) fim/fim_4_5_11_1/hand_datasets\n" + "In the Configuration Tab, click on the `Environment variables` (left menu)\n", + "- change the `FIM_VERSION` to latest publie version being used. (numerics only): ie: 5.1.0\n", + "- change the `HAND_VERSION` to latest HAND model version being used. (numerics only): ie: 4.5.11.1\n" ] }, { "cell_type": "markdown", - "id": "c2565d24-2dc5-4240-ae37-fa5374a660d4", + "id": "9d71e761-9421-49fc-aaed-aafb33b7de0a", "metadata": { "tags": [] }, @@ -418,7 +355,8 @@ "\n", "Lambda name: hv-vpp-ti-viz-hand-fim-processing\n", "\n", - "In the `Configuration` Tab, click on the `Environment variables` (left menu), then change the `FIM_VERSION` to the latest fim model version. \n", + "In the `Configuration` Tab, click on the `Environment variables` (left menu):\n", + "- change the `FIM_VERSION` to the latest fim model version. \n", "
\n", "ie) 4.5.11.1\n", "

\n", @@ -427,12 +365,7 @@ "
Change (if they are not already there)\n", "
Memory (text field) to 4096 (MB) and\n", "
Emphermeral Storage tp 1024 (MB)\n", - "
\n", - "\n", - "#### Note: Later in these steps we will change the Memory and Emphermal Storage back to default values, see below ####\n", - "Nov 5, 2024: Added new variable called \"MODEL_VERSION\" : \"4.5.11.1\". --- FIM_VERSION IS NOW: (blank). field might be removed later)\n", - "\n", - "\n" + "
\n" ] }, { @@ -457,21 +390,12 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "dfc0a44c-8b46-4db5-a9a0-c65e0bf1e20b", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Updating geocurves table to model_version of 2.0 to follow the new versioning system and drop fim_version\n", - "Updating done for geocurves\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Update \"geocurves\" to update the \"fim_version\" field to \"FIM 5.1.0:\n", @@ -522,7 +446,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 7, "id": "6a698067", "metadata": { "tags": [] @@ -591,7 +515,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 8, "id": "f6d6ee69", "metadata": { "tags": [] @@ -601,7 +525,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 45 mins. Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_2_20241111T1720\n" + "AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 15 - 45 mins.\n", + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_2_20241116T1902\n" ] } ], @@ -611,7 +536,7 @@ "pipeline_input = get_aep_pipeline_input(\"2\")\n", "\n", "# notice, slightly different object name\n", - "pipeline_name = f\"sagemaker_aep_2_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_sage_aep_2_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "STEPFUNCTION_CLIENT.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", @@ -619,13 +544,14 @@ " input= json.dumps(pipeline_input)\n", ")\n", "\n", - "print(\"AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 15 - 45 mins.\"\n", - "print(f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: {pipeline_name}\")\n" + "print(\"AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 15 - 45 mins.\")\n", + "print(f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: {pipeline_name}\")\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 9, "id": "a4f89d9a", "metadata": { "tags": [] @@ -636,7 +562,7 @@ "output_type": "stream", "text": [ "AEP : 5 year flows ie: rf_5_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_5_20241111T1743\n" + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_5_20241116T1921\n" ] } ], @@ -646,7 +572,7 @@ "pipeline_input = get_aep_pipeline_input(\"5\")\n", "\n", "# notice, slightly different object name\n", - "pipeline_name = f\"sagemaker_aep_5_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_sage_aep_5_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "STEPFUNCTION_CLIENT.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", @@ -660,13 +586,9 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 10, "id": "791d1a8b", "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, "tags": [] }, "outputs": [ @@ -675,7 +597,7 @@ "output_type": "stream", "text": [ "AEP : 10 year flows ie: rf_10_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_10_20241111T1743\n" + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_10_20241116T1921\n" ] } ], @@ -685,7 +607,7 @@ "pipeline_input = get_aep_pipeline_input(\"10\")\n", "\n", "# notice, slightly different object name\n", - "pipeline_name = f\"sagemaker_aep_10_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_sage_aep_10_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "STEPFUNCTION_CLIENT.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", @@ -699,13 +621,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 11, "id": "1bb87128", "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, "tags": [] }, "outputs": [ @@ -714,7 +632,7 @@ "output_type": "stream", "text": [ "AEP : 25 year flows ie: rf_25_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_25_20241111T1743\n" + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_25_20241116T1921\n" ] } ], @@ -724,7 +642,7 @@ "pipeline_input = get_aep_pipeline_input(\"25\")\n", "\n", "# notice, slightly different object name\n", - "pipeline_name = f\"sagemaker_aep_25_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_sage_aep_25_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "STEPFUNCTION_CLIENT.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", @@ -738,13 +656,9 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 12, "id": "4832e4e0", "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, "tags": [] }, "outputs": [ @@ -753,7 +667,7 @@ "output_type": "stream", "text": [ "AEP : 50 year flows ie: rf_50_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_50_20241111T1743\n" + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_50_20241116T1921\n" ] } ], @@ -763,7 +677,7 @@ "pipeline_input = get_aep_pipeline_input(\"50\")\n", "\n", "# notice, slightly different object name\n", - "pipeline_name = f\"sagemaker_aep_50_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_sage_aep_50_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "STEPFUNCTION_CLIENT.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", @@ -777,13 +691,9 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 13, "id": "187e83bc-ebbe-4615-a046-e0ef7b09ad3b", "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, "tags": [] }, "outputs": [ @@ -792,7 +702,7 @@ "output_type": "stream", "text": [ "AEP : High Water year flows ie: rf_hw_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_hw_20241111T1743\n", + "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_hw_20241116T1921\n", "\n" ] } @@ -803,7 +713,7 @@ "pipeline_input = get_aep_pipeline_input(\"high_water\")\n", "\n", "# notice, slightly different object name\n", - "pipeline_name = f\"sagemaker_aep_hw_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_sage_aep_hw_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "STEPFUNCTION_CLIENT.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", @@ -850,7 +760,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 14, "id": "c77d6e5b-f5fb-422f-a6c3-0f53743cd631", "metadata": { "tags": [] @@ -885,7 +795,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 15, "id": "a7495759", "metadata": { "tags": [] @@ -895,7 +805,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Catchments Branch 0 load kicked off. Last runtime: 23:38.019. Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - sagemaker_0_catchments_20241111T1815\n" + "Catchments Branch 0 load kicked off. Last runtime: 23:38.019. Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - sagemaker_0_catchments_20241116T1945\n" ] } ], @@ -991,7 +901,7 @@ " }\n", "}\n", "\n", - "pipeline_name = f\"sagemaker_0_catchments_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_sage_0_catchments_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "# TODO: For later... fix fim_version value and add model_version column. current fim_version vlaue is showing 4.5.2.11\n", "\n", @@ -1135,7 +1045,7 @@ " }\n", "}\n", "\n", - "pipeline_name = f\"sagemaker_gms_catchments_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_sage_gms_catchments_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "# TODO: For later... fix fim_version value and add model_version column. current fim_version vlaue is showing 4.5.2.11\n", "\n", @@ -1161,2178 +1071,13 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Processing 1 of 1000 on page 1\n", - "Processing 2 of 1000 on page 1\n", - "Processing 3 of 1000 on page 1\n", - "Processing 4 of 1000 on page 1\n", - "Processing 5 of 1000 on page 1\n", - "Processing 6 of 1000 on page 1\n", - "Processing 7 of 1000 on page 1\n", - "Processing 8 of 1000 on page 1\n", - "Processing 9 of 1000 on page 1\n", - "Processing 10 of 1000 on page 1\n", - "Processing 11 of 1000 on page 1\n", - "Processing 12 of 1000 on page 1\n", - "Processing 13 of 1000 on page 1\n", - "Processing 14 of 1000 on page 1\n", - "Processing 15 of 1000 on page 1\n", - "Processing 16 of 1000 on page 1\n", - "Processing 17 of 1000 on page 1\n", - "Processing 18 of 1000 on page 1\n", - "Processing 19 of 1000 on page 1\n", - "Processing 20 of 1000 on page 1\n", - "Processing 21 of 1000 on page 1\n", - "Processing 22 of 1000 on page 1\n", - "Processing 23 of 1000 on page 1\n", - "Processing 24 of 1000 on page 1\n", - "Processing 25 of 1000 on page 1\n", - "Processing 26 of 1000 on page 1\n", - "Processing 27 of 1000 on page 1\n", - "Processing 28 of 1000 on page 1\n", - "Processing 29 of 1000 on page 1\n", - "Processing 30 of 1000 on page 1\n", - "Processing 31 of 1000 on page 1\n", - "Processing 32 of 1000 on page 1\n", - "Processing 33 of 1000 on page 1\n", - "Processing 34 of 1000 on page 1\n", - "Processing 35 of 1000 on page 1\n", - "Processing 36 of 1000 on page 1\n", - "Processing 37 of 1000 on page 1\n", - "Processing 38 of 1000 on page 1\n", - "Processing 39 of 1000 on page 1\n", - "Processing 40 of 1000 on page 1\n", - "Processing 41 of 1000 on page 1\n", - "Processing 42 of 1000 on page 1\n", - "Processing 43 of 1000 on page 1\n", - "Processing 44 of 1000 on page 1\n", - "Processing 45 of 1000 on page 1\n", - "Processing 46 of 1000 on page 1\n", - "Processing 47 of 1000 on page 1\n", - "Processing 48 of 1000 on page 1\n", - "Processing 49 of 1000 on page 1\n", - "Processing 50 of 1000 on page 1\n", - "Processing 51 of 1000 on page 1\n", - "Processing 52 of 1000 on page 1\n", - "Processing 53 of 1000 on page 1\n", - "Processing 54 of 1000 on page 1\n", - "Processing 55 of 1000 on page 1\n", - "Processing 56 of 1000 on page 1\n", - "Processing 57 of 1000 on page 1\n", - "Processing 58 of 1000 on page 1\n", - "Processing 59 of 1000 on page 1\n", - "Processing 60 of 1000 on page 1\n", - "Processing 61 of 1000 on page 1\n", - "Processing 62 of 1000 on page 1\n", - "Processing 63 of 1000 on page 1\n", - "Processing 64 of 1000 on page 1\n", - "Processing 65 of 1000 on page 1\n", - "Processing 66 of 1000 on page 1\n", - "Processing 67 of 1000 on page 1\n", - "Processing 68 of 1000 on page 1\n", - "Processing 69 of 1000 on page 1\n", - "Processing 70 of 1000 on page 1\n", - "Processing 71 of 1000 on page 1\n", - "Processing 72 of 1000 on page 1\n", - "Processing 73 of 1000 on page 1\n", - "Processing 74 of 1000 on page 1\n", - "Processing 75 of 1000 on page 1\n", - "Processing 76 of 1000 on page 1\n", - "Processing 77 of 1000 on page 1\n", - "Processing 78 of 1000 on page 1\n", - "Processing 79 of 1000 on page 1\n", - "Processing 80 of 1000 on page 1\n", - "Processing 81 of 1000 on page 1\n", - "Processing 82 of 1000 on page 1\n", - "Processing 83 of 1000 on page 1\n", - "Processing 84 of 1000 on page 1\n", - "Processing 85 of 1000 on page 1\n", - "Processing 86 of 1000 on page 1\n", - "Processing 87 of 1000 on page 1\n", - "Processing 88 of 1000 on page 1\n", - "Processing 89 of 1000 on page 1\n", - "Processing 90 of 1000 on page 1\n", - "Processing 91 of 1000 on page 1\n", - "Processing 92 of 1000 on page 1\n", - "Processing 93 of 1000 on page 1\n", - "Processing 94 of 1000 on page 1\n", - "Processing 95 of 1000 on page 1\n", - "Processing 96 of 1000 on page 1\n", - "Processing 97 of 1000 on page 1\n", - "Processing 98 of 1000 on page 1\n", - "Processing 99 of 1000 on page 1\n", - "Processing 100 of 1000 on page 1\n", - "Processing 101 of 1000 on page 1\n", - "Processing 102 of 1000 on page 1\n", - "Processing 103 of 1000 on page 1\n", - "Processing 104 of 1000 on page 1\n", - "Processing 105 of 1000 on page 1\n", - "Processing 106 of 1000 on page 1\n", - "Processing 107 of 1000 on page 1\n", - "Processing 108 of 1000 on page 1\n", - "Processing 109 of 1000 on page 1\n", - "Processing 110 of 1000 on page 1\n", - "Processing 111 of 1000 on page 1\n", - "Processing 112 of 1000 on page 1\n", - "Processing 113 of 1000 on page 1\n", - "Processing 114 of 1000 on page 1\n", - "Processing 115 of 1000 on page 1\n", - "Processing 116 of 1000 on page 1\n", - "Processing 117 of 1000 on page 1\n", - "Processing 118 of 1000 on page 1\n", - "Processing 119 of 1000 on page 1\n", - "Processing 120 of 1000 on page 1\n", - "Processing 121 of 1000 on page 1\n", - "Processing 122 of 1000 on page 1\n", - "Processing 123 of 1000 on page 1\n", - "Processing 124 of 1000 on page 1\n", - "Processing 125 of 1000 on page 1\n", - "Processing 126 of 1000 on page 1\n", - "Processing 127 of 1000 on page 1\n", - "Processing 128 of 1000 on page 1\n", - "Processing 129 of 1000 on page 1\n", - "Processing 130 of 1000 on page 1\n", - "Processing 131 of 1000 on page 1\n", - "Processing 132 of 1000 on page 1\n", - "Processing 133 of 1000 on page 1\n", - "Processing 134 of 1000 on page 1\n", - "Processing 135 of 1000 on page 1\n", - "Processing 136 of 1000 on page 1\n", - "Processing 137 of 1000 on page 1\n", - "Processing 138 of 1000 on page 1\n", - "Processing 139 of 1000 on page 1\n", - "Processing 140 of 1000 on page 1\n", - "Processing 141 of 1000 on page 1\n", - "Processing 142 of 1000 on page 1\n", - "Processing 143 of 1000 on page 1\n", - "Processing 144 of 1000 on page 1\n", - "Processing 145 of 1000 on page 1\n", - "Processing 146 of 1000 on page 1\n", - "Processing 147 of 1000 on page 1\n", - "Processing 148 of 1000 on page 1\n", - "Processing 149 of 1000 on page 1\n", - "Processing 150 of 1000 on page 1\n", - "Processing 151 of 1000 on page 1\n", - "Processing 152 of 1000 on page 1\n", - "Processing 153 of 1000 on page 1\n", - "Processing 154 of 1000 on page 1\n", - "Processing 155 of 1000 on page 1\n", - "Processing 156 of 1000 on page 1\n", - "Processing 157 of 1000 on page 1\n", - "Processing 158 of 1000 on page 1\n", - "Processing 159 of 1000 on page 1\n", - "Processing 160 of 1000 on page 1\n", - "Processing 161 of 1000 on page 1\n", - "Processing 162 of 1000 on page 1\n", - "Processing 163 of 1000 on page 1\n", - "Processing 164 of 1000 on page 1\n", - "Processing 165 of 1000 on page 1\n", - "Processing 166 of 1000 on page 1\n", - "Processing 167 of 1000 on page 1\n", - "Processing 168 of 1000 on page 1\n", - "Processing 169 of 1000 on page 1\n", - "Processing 170 of 1000 on page 1\n", - "Processing 171 of 1000 on page 1\n", - "Processing 172 of 1000 on page 1\n", - "Processing 173 of 1000 on page 1\n", - "Processing 174 of 1000 on page 1\n", - "Processing 175 of 1000 on page 1\n", - "Processing 176 of 1000 on page 1\n", - "Processing 177 of 1000 on page 1\n", - "Processing 178 of 1000 on page 1\n", - "Processing 179 of 1000 on page 1\n", - "Processing 180 of 1000 on page 1\n", - "Processing 181 of 1000 on page 1\n", - "Processing 182 of 1000 on page 1\n", - "Processing 183 of 1000 on page 1\n", - "Processing 184 of 1000 on page 1\n", - "Processing 185 of 1000 on page 1\n", - "Processing 186 of 1000 on page 1\n", - "Processing 187 of 1000 on page 1\n", - "Processing 188 of 1000 on page 1\n", - "Processing 189 of 1000 on page 1\n", - "Processing 190 of 1000 on page 1\n", - "Processing 191 of 1000 on page 1\n", - "Processing 192 of 1000 on page 1\n", - "Processing 193 of 1000 on page 1\n", - "Processing 194 of 1000 on page 1\n", - "Processing 195 of 1000 on page 1\n", - "Processing 196 of 1000 on page 1\n", - "Processing 197 of 1000 on page 1\n", - "Processing 198 of 1000 on page 1\n", - "Processing 199 of 1000 on page 1\n", - "Processing 200 of 1000 on page 1\n", - "Processing 201 of 1000 on page 1\n", - "Processing 202 of 1000 on page 1\n", - "Processing 203 of 1000 on page 1\n", - "Processing 204 of 1000 on page 1\n", - "Processing 205 of 1000 on page 1\n", - "Processing 206 of 1000 on page 1\n", - "Processing 207 of 1000 on page 1\n", - "Processing 208 of 1000 on page 1\n", - "Processing 209 of 1000 on page 1\n", - "Processing 210 of 1000 on page 1\n", - "Processing 211 of 1000 on page 1\n", - "Processing 212 of 1000 on page 1\n", - "Processing 213 of 1000 on page 1\n", - "Processing 214 of 1000 on page 1\n", - "Processing 215 of 1000 on page 1\n", - "Processing 216 of 1000 on page 1\n", - "Processing 217 of 1000 on page 1\n", - "Processing 218 of 1000 on page 1\n", - "Processing 219 of 1000 on page 1\n", - "Processing 220 of 1000 on page 1\n", - "Processing 221 of 1000 on page 1\n", - "Processing 222 of 1000 on page 1\n", - "Processing 223 of 1000 on page 1\n", - "Processing 224 of 1000 on page 1\n", - "Processing 225 of 1000 on page 1\n", - "Processing 226 of 1000 on page 1\n", - "Processing 227 of 1000 on page 1\n", - "Processing 228 of 1000 on page 1\n", - "Processing 229 of 1000 on page 1\n", - "Processing 230 of 1000 on page 1\n", - "Processing 231 of 1000 on page 1\n", - "Processing 232 of 1000 on page 1\n", - "Processing 233 of 1000 on page 1\n", - "Processing 234 of 1000 on page 1\n", - "Processing 235 of 1000 on page 1\n", - "Processing 236 of 1000 on page 1\n", - "Processing 237 of 1000 on page 1\n", - "Processing 238 of 1000 on page 1\n", - "Processing 239 of 1000 on page 1\n", - "Processing 240 of 1000 on page 1\n", - "Processing 241 of 1000 on page 1\n", - "Processing 242 of 1000 on page 1\n", - "Processing 243 of 1000 on page 1\n", - "Processing 244 of 1000 on page 1\n", - "Processing 245 of 1000 on page 1\n", - "Processing 246 of 1000 on page 1\n", - "Processing 247 of 1000 on page 1\n", - "Processing 248 of 1000 on page 1\n", - "Processing 249 of 1000 on page 1\n", - "Processing 250 of 1000 on page 1\n", - "Processing 251 of 1000 on page 1\n", - "Processing 252 of 1000 on page 1\n", - "Processing 253 of 1000 on page 1\n", - "Processing 254 of 1000 on page 1\n", - "Processing 255 of 1000 on page 1\n", - "Processing 256 of 1000 on page 1\n", - "Processing 257 of 1000 on page 1\n", - "Processing 258 of 1000 on page 1\n", - "Processing 259 of 1000 on page 1\n", - "Processing 260 of 1000 on page 1\n", - "Processing 261 of 1000 on page 1\n", - "Processing 262 of 1000 on page 1\n", - "Processing 263 of 1000 on page 1\n", - "Processing 264 of 1000 on page 1\n", - "Processing 265 of 1000 on page 1\n", - "Processing 266 of 1000 on page 1\n", - "Processing 267 of 1000 on page 1\n", - "Processing 268 of 1000 on page 1\n", - "Processing 269 of 1000 on page 1\n", - "Processing 270 of 1000 on page 1\n", - "Processing 271 of 1000 on page 1\n", - "Processing 272 of 1000 on page 1\n", - "Processing 273 of 1000 on page 1\n", - "Processing 274 of 1000 on page 1\n", - "Processing 275 of 1000 on page 1\n", - "Processing 276 of 1000 on page 1\n", - "Processing 277 of 1000 on page 1\n", - "Processing 278 of 1000 on page 1\n", - "Processing 279 of 1000 on page 1\n", - "Processing 280 of 1000 on page 1\n", - "Processing 281 of 1000 on page 1\n", - "Processing 282 of 1000 on page 1\n", - "Processing 283 of 1000 on page 1\n", - "Processing 284 of 1000 on page 1\n", - "Processing 285 of 1000 on page 1\n", - "Processing 286 of 1000 on page 1\n", - "Processing 287 of 1000 on page 1\n", - "Processing 288 of 1000 on page 1\n", - "Processing 289 of 1000 on page 1\n", - "Processing 290 of 1000 on page 1\n", - "Processing 291 of 1000 on page 1\n", - "Processing 292 of 1000 on page 1\n", - "Processing 293 of 1000 on page 1\n", - "Processing 294 of 1000 on page 1\n", - "Processing 295 of 1000 on page 1\n", - "Processing 296 of 1000 on page 1\n", - "Processing 297 of 1000 on page 1\n", - "Processing 298 of 1000 on page 1\n", - "Processing 299 of 1000 on page 1\n", - "Processing 300 of 1000 on page 1\n", - "Processing 301 of 1000 on page 1\n", - "Processing 302 of 1000 on page 1\n", - "Processing 303 of 1000 on page 1\n", - "Processing 304 of 1000 on page 1\n", - "Processing 305 of 1000 on page 1\n", - "Processing 306 of 1000 on page 1\n", - "Processing 307 of 1000 on page 1\n", - "Processing 308 of 1000 on page 1\n", - "Processing 309 of 1000 on page 1\n", - "Processing 310 of 1000 on page 1\n", - "Processing 311 of 1000 on page 1\n", - "Processing 312 of 1000 on page 1\n", - "Processing 313 of 1000 on page 1\n", - "Processing 314 of 1000 on page 1\n", - "Processing 315 of 1000 on page 1\n", - "Processing 316 of 1000 on page 1\n", - "Processing 317 of 1000 on page 1\n", - "Processing 318 of 1000 on page 1\n", - "Processing 319 of 1000 on page 1\n", - "Processing 320 of 1000 on page 1\n", - "Processing 321 of 1000 on page 1\n", - "Processing 322 of 1000 on page 1\n", - "Processing 323 of 1000 on page 1\n", - "Processing 324 of 1000 on page 1\n", - "Processing 325 of 1000 on page 1\n", - "Processing 326 of 1000 on page 1\n", - "Processing 327 of 1000 on page 1\n", - "Processing 328 of 1000 on page 1\n", - "Processing 329 of 1000 on page 1\n", - "Processing 330 of 1000 on page 1\n", - "Processing 331 of 1000 on page 1\n", - "Processing 332 of 1000 on page 1\n", - "Processing 333 of 1000 on page 1\n", - "Processing 334 of 1000 on page 1\n", - "Processing 335 of 1000 on page 1\n", - "Processing 336 of 1000 on page 1\n", - "Processing 337 of 1000 on page 1\n", - "Processing 338 of 1000 on page 1\n", - "Processing 339 of 1000 on page 1\n", - "Processing 340 of 1000 on page 1\n", - "Processing 341 of 1000 on page 1\n", - "Processing 342 of 1000 on page 1\n", - "Processing 343 of 1000 on page 1\n", - "Processing 344 of 1000 on page 1\n", - "Processing 345 of 1000 on page 1\n", - "Processing 346 of 1000 on page 1\n", - "Processing 347 of 1000 on page 1\n", - "Processing 348 of 1000 on page 1\n", - "Processing 349 of 1000 on page 1\n", - "Processing 350 of 1000 on page 1\n", - "Processing 351 of 1000 on page 1\n", - "Processing 352 of 1000 on page 1\n", - "Processing 353 of 1000 on page 1\n", - "Processing 354 of 1000 on page 1\n", - "Processing 355 of 1000 on page 1\n", - "Processing 356 of 1000 on page 1\n", - "Processing 357 of 1000 on page 1\n", - "Processing 358 of 1000 on page 1\n", - "Processing 359 of 1000 on page 1\n", - "Processing 360 of 1000 on page 1\n", - "Processing 361 of 1000 on page 1\n", - "Processing 362 of 1000 on page 1\n", - "Processing 363 of 1000 on page 1\n", - "Processing 364 of 1000 on page 1\n", - "Processing 365 of 1000 on page 1\n", - "Processing 366 of 1000 on page 1\n", - "Processing 367 of 1000 on page 1\n", - "Processing 368 of 1000 on page 1\n", - "Processing 369 of 1000 on page 1\n", - "Processing 370 of 1000 on page 1\n", - "Processing 371 of 1000 on page 1\n", - "Processing 372 of 1000 on page 1\n", - "Processing 373 of 1000 on page 1\n", - "Processing 374 of 1000 on page 1\n", - "Processing 375 of 1000 on page 1\n", - "Processing 376 of 1000 on page 1\n", - "Processing 377 of 1000 on page 1\n", - "Processing 378 of 1000 on page 1\n", - "Processing 379 of 1000 on page 1\n", - "Processing 380 of 1000 on page 1\n", - "Processing 381 of 1000 on page 1\n", - "Processing 382 of 1000 on page 1\n", - "Processing 383 of 1000 on page 1\n", - "Processing 384 of 1000 on page 1\n", - "Processing 385 of 1000 on page 1\n", - "Processing 386 of 1000 on page 1\n", - "Processing 387 of 1000 on page 1\n", - "Processing 388 of 1000 on page 1\n", - "Processing 389 of 1000 on page 1\n", - "Processing 390 of 1000 on page 1\n", - "Processing 391 of 1000 on page 1\n", - "Processing 392 of 1000 on page 1\n", - "Processing 393 of 1000 on page 1\n", - "Processing 394 of 1000 on page 1\n", - "Processing 395 of 1000 on page 1\n", - "Processing 396 of 1000 on page 1\n", - "Processing 397 of 1000 on page 1\n", - "Processing 398 of 1000 on page 1\n", - "Processing 399 of 1000 on page 1\n", - "Processing 400 of 1000 on page 1\n", - "Processing 401 of 1000 on page 1\n", - "Processing 402 of 1000 on page 1\n", - "Processing 403 of 1000 on page 1\n", - "Processing 404 of 1000 on page 1\n", - "Processing 405 of 1000 on page 1\n", - "Processing 406 of 1000 on page 1\n", - "Processing 407 of 1000 on page 1\n", - "Processing 408 of 1000 on page 1\n", - "Processing 409 of 1000 on page 1\n", - "Processing 410 of 1000 on page 1\n", - "Processing 411 of 1000 on page 1\n", - "Processing 412 of 1000 on page 1\n", - "Processing 413 of 1000 on page 1\n", - "Processing 414 of 1000 on page 1\n", - "Processing 415 of 1000 on page 1\n", - "Processing 416 of 1000 on page 1\n", - "Processing 417 of 1000 on page 1\n", - "Processing 418 of 1000 on page 1\n", - "Processing 419 of 1000 on page 1\n", - "Processing 420 of 1000 on page 1\n", - "Processing 421 of 1000 on page 1\n", - "Processing 422 of 1000 on page 1\n", - "Processing 423 of 1000 on page 1\n", - "Processing 424 of 1000 on page 1\n", - "Processing 425 of 1000 on page 1\n", - "Processing 426 of 1000 on page 1\n", - "Processing 427 of 1000 on page 1\n", - "Processing 428 of 1000 on page 1\n", - "Processing 429 of 1000 on page 1\n", - "Processing 430 of 1000 on page 1\n", - "Processing 431 of 1000 on page 1\n", - "Processing 432 of 1000 on page 1\n", - "Processing 433 of 1000 on page 1\n", - "Processing 434 of 1000 on page 1\n", - "Processing 435 of 1000 on page 1\n", - "Processing 436 of 1000 on page 1\n", - "Processing 437 of 1000 on page 1\n", - "Processing 438 of 1000 on page 1\n", - "Processing 439 of 1000 on page 1\n", - "Processing 440 of 1000 on page 1\n", - "Processing 441 of 1000 on page 1\n", - "Processing 442 of 1000 on page 1\n", - "Processing 443 of 1000 on page 1\n", - "Processing 444 of 1000 on page 1\n", - "Processing 445 of 1000 on page 1\n", - "Processing 446 of 1000 on page 1\n", - "Processing 447 of 1000 on page 1\n", - "Processing 448 of 1000 on page 1\n", - "Processing 449 of 1000 on page 1\n", - "Processing 450 of 1000 on page 1\n", - "Processing 451 of 1000 on page 1\n", - "Processing 452 of 1000 on page 1\n", - "Processing 453 of 1000 on page 1\n", - "Processing 454 of 1000 on page 1\n", - "Processing 455 of 1000 on page 1\n", - "Processing 456 of 1000 on page 1\n", - "Processing 457 of 1000 on page 1\n", - "Processing 458 of 1000 on page 1\n", - "Processing 459 of 1000 on page 1\n", - "Processing 460 of 1000 on page 1\n", - "Processing 461 of 1000 on page 1\n", - "Processing 462 of 1000 on page 1\n", - "Processing 463 of 1000 on page 1\n", - "Processing 464 of 1000 on page 1\n", - "Processing 465 of 1000 on page 1\n", - "Processing 466 of 1000 on page 1\n", - "Processing 467 of 1000 on page 1\n", - "Processing 468 of 1000 on page 1\n", - "Processing 469 of 1000 on page 1\n", - "Processing 470 of 1000 on page 1\n", - "Processing 471 of 1000 on page 1\n", - "Processing 472 of 1000 on page 1\n", - "Processing 473 of 1000 on page 1\n", - "Processing 474 of 1000 on page 1\n", - "Processing 475 of 1000 on page 1\n", - "Processing 476 of 1000 on page 1\n", - "Processing 477 of 1000 on page 1\n", - "Processing 478 of 1000 on page 1\n", - "Processing 479 of 1000 on page 1\n", - "Processing 480 of 1000 on page 1\n", - "Processing 481 of 1000 on page 1\n", - "Processing 482 of 1000 on page 1\n", - "Processing 483 of 1000 on page 1\n", - "Processing 484 of 1000 on page 1\n", - "Processing 485 of 1000 on page 1\n", - "Processing 486 of 1000 on page 1\n", - "Processing 487 of 1000 on page 1\n", - "Processing 488 of 1000 on page 1\n", - "Processing 489 of 1000 on page 1\n", - "Processing 490 of 1000 on page 1\n", - "Processing 491 of 1000 on page 1\n", - "Processing 492 of 1000 on page 1\n", - "Processing 493 of 1000 on page 1\n", - "Processing 494 of 1000 on page 1\n", - "Processing 495 of 1000 on page 1\n", - "Processing 496 of 1000 on page 1\n", - "Processing 497 of 1000 on page 1\n", - "Processing 498 of 1000 on page 1\n", - "Processing 499 of 1000 on page 1\n", - "Processing 500 of 1000 on page 1\n", - "Processing 501 of 1000 on page 1\n", - "Processing 502 of 1000 on page 1\n", - "Processing 503 of 1000 on page 1\n", - "Processing 504 of 1000 on page 1\n", - "Processing 505 of 1000 on page 1\n", - "Processing 506 of 1000 on page 1\n", - "Processing 507 of 1000 on page 1\n", - "Processing 508 of 1000 on page 1\n", - "Processing 509 of 1000 on page 1\n", - "Processing 510 of 1000 on page 1\n", - "Processing 511 of 1000 on page 1\n", - "Processing 512 of 1000 on page 1\n", - "Processing 513 of 1000 on page 1\n", - "Processing 514 of 1000 on page 1\n", - "Processing 515 of 1000 on page 1\n", - "Processing 516 of 1000 on page 1\n", - "Processing 517 of 1000 on page 1\n", - "Processing 518 of 1000 on page 1\n", - "Processing 519 of 1000 on page 1\n", - "Processing 520 of 1000 on page 1\n", - "Processing 521 of 1000 on page 1\n", - "Processing 522 of 1000 on page 1\n", - "Processing 523 of 1000 on page 1\n", - "Processing 524 of 1000 on page 1\n", - "Processing 525 of 1000 on page 1\n", - "Processing 526 of 1000 on page 1\n", - "Processing 527 of 1000 on page 1\n", - "Processing 528 of 1000 on page 1\n", - "Processing 529 of 1000 on page 1\n", - "Processing 530 of 1000 on page 1\n", - "Processing 531 of 1000 on page 1\n", - "Processing 532 of 1000 on page 1\n", - "Processing 533 of 1000 on page 1\n", - "Processing 534 of 1000 on page 1\n", - "Processing 535 of 1000 on page 1\n", - "Processing 536 of 1000 on page 1\n", - "Processing 537 of 1000 on page 1\n", - "Processing 538 of 1000 on page 1\n", - "Processing 539 of 1000 on page 1\n", - "Processing 540 of 1000 on page 1\n", - "Processing 541 of 1000 on page 1\n", - "Processing 542 of 1000 on page 1\n", - "Processing 543 of 1000 on page 1\n", - "Processing 544 of 1000 on page 1\n", - "Processing 545 of 1000 on page 1\n", - "Processing 546 of 1000 on page 1\n", - "Processing 547 of 1000 on page 1\n", - "Processing 548 of 1000 on page 1\n", - "Processing 549 of 1000 on page 1\n", - "Processing 550 of 1000 on page 1\n", - "Processing 551 of 1000 on page 1\n", - "Processing 552 of 1000 on page 1\n", - "Processing 553 of 1000 on page 1\n", - "Processing 554 of 1000 on page 1\n", - "Processing 555 of 1000 on page 1\n", - "Processing 556 of 1000 on page 1\n", - "Processing 557 of 1000 on page 1\n", - "Processing 558 of 1000 on page 1\n", - "Processing 559 of 1000 on page 1\n", - "Processing 560 of 1000 on page 1\n", - "Processing 561 of 1000 on page 1\n", - "Processing 562 of 1000 on page 1\n", - "Processing 563 of 1000 on page 1\n", - "Processing 564 of 1000 on page 1\n", - "Processing 565 of 1000 on page 1\n", - "Processing 566 of 1000 on page 1\n", - "Processing 567 of 1000 on page 1\n", - "Processing 568 of 1000 on page 1\n", - "Processing 569 of 1000 on page 1\n", - "Processing 570 of 1000 on page 1\n", - "Processing 571 of 1000 on page 1\n", - "Processing 572 of 1000 on page 1\n", - "Processing 573 of 1000 on page 1\n", - "Processing 574 of 1000 on page 1\n", - "Processing 575 of 1000 on page 1\n", - "Processing 576 of 1000 on page 1\n", - "Processing 577 of 1000 on page 1\n", - "Processing 578 of 1000 on page 1\n", - "Processing 579 of 1000 on page 1\n", - "Processing 580 of 1000 on page 1\n", - "Processing 581 of 1000 on page 1\n", - "Processing 582 of 1000 on page 1\n", - "Processing 583 of 1000 on page 1\n", - "Processing 584 of 1000 on page 1\n", - "Processing 585 of 1000 on page 1\n", - "Processing 586 of 1000 on page 1\n", - "Processing 587 of 1000 on page 1\n", - "Processing 588 of 1000 on page 1\n", - "Processing 589 of 1000 on page 1\n", - "Processing 590 of 1000 on page 1\n", - "Processing 591 of 1000 on page 1\n", - "Processing 592 of 1000 on page 1\n", - "Processing 593 of 1000 on page 1\n", - "Processing 594 of 1000 on page 1\n", - "Processing 595 of 1000 on page 1\n", - "Processing 596 of 1000 on page 1\n", - "Processing 597 of 1000 on page 1\n", - "Processing 598 of 1000 on page 1\n", - "Processing 599 of 1000 on page 1\n", - "Processing 600 of 1000 on page 1\n", - "Processing 601 of 1000 on page 1\n", - "Processing 602 of 1000 on page 1\n", - "Processing 603 of 1000 on page 1\n", - "Processing 604 of 1000 on page 1\n", - "Processing 605 of 1000 on page 1\n", - "Processing 606 of 1000 on page 1\n", - "Processing 607 of 1000 on page 1\n", - "Processing 608 of 1000 on page 1\n", - "Processing 609 of 1000 on page 1\n", - "Processing 610 of 1000 on page 1\n", - "Processing 611 of 1000 on page 1\n", - "Processing 612 of 1000 on page 1\n", - "Processing 613 of 1000 on page 1\n", - "Processing 614 of 1000 on page 1\n", - "Processing 615 of 1000 on page 1\n", - "Processing 616 of 1000 on page 1\n", - "Processing 617 of 1000 on page 1\n", - "Processing 618 of 1000 on page 1\n", - "Processing 619 of 1000 on page 1\n", - "Processing 620 of 1000 on page 1\n", - "Processing 621 of 1000 on page 1\n", - "Processing 622 of 1000 on page 1\n", - "Processing 623 of 1000 on page 1\n", - "Processing 624 of 1000 on page 1\n", - "Processing 625 of 1000 on page 1\n", - "Processing 626 of 1000 on page 1\n", - "Processing 627 of 1000 on page 1\n", - "Processing 628 of 1000 on page 1\n", - "Processing 629 of 1000 on page 1\n", - "Processing 630 of 1000 on page 1\n", - "Processing 631 of 1000 on page 1\n", - "Processing 632 of 1000 on page 1\n", - "Processing 633 of 1000 on page 1\n", - "Processing 634 of 1000 on page 1\n", - "Processing 635 of 1000 on page 1\n", - "Processing 636 of 1000 on page 1\n", - "Processing 637 of 1000 on page 1\n", - "Processing 638 of 1000 on page 1\n", - "Processing 639 of 1000 on page 1\n", - "Processing 640 of 1000 on page 1\n", - "Processing 641 of 1000 on page 1\n", - "Processing 642 of 1000 on page 1\n", - "Processing 643 of 1000 on page 1\n", - "Processing 644 of 1000 on page 1\n", - "Processing 645 of 1000 on page 1\n", - "Processing 646 of 1000 on page 1\n", - "Processing 647 of 1000 on page 1\n", - "Processing 648 of 1000 on page 1\n", - "Processing 649 of 1000 on page 1\n", - "Processing 650 of 1000 on page 1\n", - "Processing 651 of 1000 on page 1\n", - "Processing 652 of 1000 on page 1\n", - "Processing 653 of 1000 on page 1\n", - "Processing 654 of 1000 on page 1\n", - "Processing 655 of 1000 on page 1\n", - "Processing 656 of 1000 on page 1\n", - "Processing 657 of 1000 on page 1\n", - "Processing 658 of 1000 on page 1\n", - "Processing 659 of 1000 on page 1\n", - "Processing 660 of 1000 on page 1\n", - "Processing 661 of 1000 on page 1\n", - "Processing 662 of 1000 on page 1\n", - "Processing 663 of 1000 on page 1\n", - "Processing 664 of 1000 on page 1\n", - "Processing 665 of 1000 on page 1\n", - "Processing 666 of 1000 on page 1\n", - "Processing 667 of 1000 on page 1\n", - "Processing 668 of 1000 on page 1\n", - "Processing 669 of 1000 on page 1\n", - "Processing 670 of 1000 on page 1\n", - "Processing 671 of 1000 on page 1\n", - "Processing 672 of 1000 on page 1\n", - "Processing 673 of 1000 on page 1\n", - "Processing 674 of 1000 on page 1\n", - "Processing 675 of 1000 on page 1\n", - "Processing 676 of 1000 on page 1\n", - "Processing 677 of 1000 on page 1\n", - "Processing 678 of 1000 on page 1\n", - "Processing 679 of 1000 on page 1\n", - "Processing 680 of 1000 on page 1\n", - "Processing 681 of 1000 on page 1\n", - "Processing 682 of 1000 on page 1\n", - "Processing 683 of 1000 on page 1\n", - "Processing 684 of 1000 on page 1\n", - "Processing 685 of 1000 on page 1\n", - "Processing 686 of 1000 on page 1\n", - "Processing 687 of 1000 on page 1\n", - "Processing 688 of 1000 on page 1\n", - "Processing 689 of 1000 on page 1\n", - "Processing 690 of 1000 on page 1\n", - "Processing 691 of 1000 on page 1\n", - "Processing 692 of 1000 on page 1\n", - "Processing 693 of 1000 on page 1\n", - "Processing 694 of 1000 on page 1\n", - "Processing 695 of 1000 on page 1\n", - "Processing 696 of 1000 on page 1\n", - "Processing 697 of 1000 on page 1\n", - "Processing 698 of 1000 on page 1\n", - "Processing 699 of 1000 on page 1\n", - "Processing 700 of 1000 on page 1\n", - "Processing 701 of 1000 on page 1\n", - "Processing 702 of 1000 on page 1\n", - "Processing 703 of 1000 on page 1\n", - "Processing 704 of 1000 on page 1\n", - "Processing 705 of 1000 on page 1\n", - "Processing 706 of 1000 on page 1\n", - "Processing 707 of 1000 on page 1\n", - "Processing 708 of 1000 on page 1\n", - "Processing 709 of 1000 on page 1\n", - "Processing 710 of 1000 on page 1\n", - "Processing 711 of 1000 on page 1\n", - "Processing 712 of 1000 on page 1\n", - "Processing 713 of 1000 on page 1\n", - "Processing 714 of 1000 on page 1\n", - "Processing 715 of 1000 on page 1\n", - "Processing 716 of 1000 on page 1\n", - "Processing 717 of 1000 on page 1\n", - "Processing 718 of 1000 on page 1\n", - "Processing 719 of 1000 on page 1\n", - "Processing 720 of 1000 on page 1\n", - "Processing 721 of 1000 on page 1\n", - "Processing 722 of 1000 on page 1\n", - "Processing 723 of 1000 on page 1\n", - "Processing 724 of 1000 on page 1\n", - "Processing 725 of 1000 on page 1\n", - "Processing 726 of 1000 on page 1\n", - "Processing 727 of 1000 on page 1\n", - "Processing 728 of 1000 on page 1\n", - "Processing 729 of 1000 on page 1\n", - "Processing 730 of 1000 on page 1\n", - "Processing 731 of 1000 on page 1\n", - "Processing 732 of 1000 on page 1\n", - "Processing 733 of 1000 on page 1\n", - "Processing 734 of 1000 on page 1\n", - "Processing 735 of 1000 on page 1\n", - "Processing 736 of 1000 on page 1\n", - "Processing 737 of 1000 on page 1\n", - "Processing 738 of 1000 on page 1\n", - "Processing 739 of 1000 on page 1\n", - "Processing 740 of 1000 on page 1\n", - "Processing 741 of 1000 on page 1\n", - "Processing 742 of 1000 on page 1\n", - "Processing 743 of 1000 on page 1\n", - "Processing 744 of 1000 on page 1\n", - "Processing 745 of 1000 on page 1\n", - "Processing 746 of 1000 on page 1\n", - "Processing 747 of 1000 on page 1\n", - "Processing 748 of 1000 on page 1\n", - "Processing 749 of 1000 on page 1\n", - "Processing 750 of 1000 on page 1\n", - "Processing 751 of 1000 on page 1\n", - "Processing 752 of 1000 on page 1\n", - "Processing 753 of 1000 on page 1\n", - "Processing 754 of 1000 on page 1\n", - "Processing 755 of 1000 on page 1\n", - "Processing 756 of 1000 on page 1\n", - "Processing 757 of 1000 on page 1\n", - "Processing 758 of 1000 on page 1\n", - "Processing 759 of 1000 on page 1\n", - "Processing 760 of 1000 on page 1\n", - "Processing 761 of 1000 on page 1\n", - "Processing 762 of 1000 on page 1\n", - "Processing 763 of 1000 on page 1\n", - "Processing 764 of 1000 on page 1\n", - "Processing 765 of 1000 on page 1\n", - "Processing 766 of 1000 on page 1\n", - "Processing 767 of 1000 on page 1\n", - "Processing 768 of 1000 on page 1\n", - "Processing 769 of 1000 on page 1\n", - "Processing 770 of 1000 on page 1\n", - "Processing 771 of 1000 on page 1\n", - "Processing 772 of 1000 on page 1\n", - "Processing 773 of 1000 on page 1\n", - "Processing 774 of 1000 on page 1\n", - "Processing 775 of 1000 on page 1\n", - "Processing 776 of 1000 on page 1\n", - "Processing 777 of 1000 on page 1\n", - "Processing 778 of 1000 on page 1\n", - "Processing 779 of 1000 on page 1\n", - "Processing 780 of 1000 on page 1\n", - "Processing 781 of 1000 on page 1\n", - "Processing 782 of 1000 on page 1\n", - "Processing 783 of 1000 on page 1\n", - "Processing 784 of 1000 on page 1\n", - "Processing 785 of 1000 on page 1\n", - "Processing 786 of 1000 on page 1\n", - "Processing 787 of 1000 on page 1\n", - "Processing 788 of 1000 on page 1\n", - "Processing 789 of 1000 on page 1\n", - "Processing 790 of 1000 on page 1\n", - "Processing 791 of 1000 on page 1\n", - "Processing 792 of 1000 on page 1\n", - "Processing 793 of 1000 on page 1\n", - "Processing 794 of 1000 on page 1\n", - "Processing 795 of 1000 on page 1\n", - "Processing 796 of 1000 on page 1\n", - "Processing 797 of 1000 on page 1\n", - "Processing 798 of 1000 on page 1\n", - "Processing 799 of 1000 on page 1\n", - "Processing 800 of 1000 on page 1\n", - "Processing 801 of 1000 on page 1\n", - "Processing 802 of 1000 on page 1\n", - "Processing 803 of 1000 on page 1\n", - "Processing 804 of 1000 on page 1\n", - "Processing 805 of 1000 on page 1\n", - "Processing 806 of 1000 on page 1\n", - "Processing 807 of 1000 on page 1\n", - "Processing 808 of 1000 on page 1\n", - "Processing 809 of 1000 on page 1\n", - "Processing 810 of 1000 on page 1\n", - "Processing 811 of 1000 on page 1\n", - "Processing 812 of 1000 on page 1\n", - "Processing 813 of 1000 on page 1\n", - "Processing 814 of 1000 on page 1\n", - "Processing 815 of 1000 on page 1\n", - "Processing 816 of 1000 on page 1\n", - "Processing 817 of 1000 on page 1\n", - "Processing 818 of 1000 on page 1\n", - "Processing 819 of 1000 on page 1\n", - "Processing 820 of 1000 on page 1\n", - "Processing 821 of 1000 on page 1\n", - "Processing 822 of 1000 on page 1\n", - "Processing 823 of 1000 on page 1\n", - "Processing 824 of 1000 on page 1\n", - "Processing 825 of 1000 on page 1\n", - "Processing 826 of 1000 on page 1\n", - "Processing 827 of 1000 on page 1\n", - "Processing 828 of 1000 on page 1\n", - "Processing 829 of 1000 on page 1\n", - "Processing 830 of 1000 on page 1\n", - "Processing 831 of 1000 on page 1\n", - "Processing 832 of 1000 on page 1\n", - "Processing 833 of 1000 on page 1\n", - "Processing 834 of 1000 on page 1\n", - "Processing 835 of 1000 on page 1\n", - "Processing 836 of 1000 on page 1\n", - "Processing 837 of 1000 on page 1\n", - "Processing 838 of 1000 on page 1\n", - "Processing 839 of 1000 on page 1\n", - "Processing 840 of 1000 on page 1\n", - "Processing 841 of 1000 on page 1\n", - "Processing 842 of 1000 on page 1\n", - "Processing 843 of 1000 on page 1\n", - "Processing 844 of 1000 on page 1\n", - "Processing 845 of 1000 on page 1\n", - "Processing 846 of 1000 on page 1\n", - "Processing 847 of 1000 on page 1\n", - "Processing 848 of 1000 on page 1\n", - "Processing 849 of 1000 on page 1\n", - "Processing 850 of 1000 on page 1\n", - "Processing 851 of 1000 on page 1\n", - "Processing 852 of 1000 on page 1\n", - "Processing 853 of 1000 on page 1\n", - "Processing 854 of 1000 on page 1\n", - "Processing 855 of 1000 on page 1\n", - "Processing 856 of 1000 on page 1\n", - "Processing 857 of 1000 on page 1\n", - "Processing 858 of 1000 on page 1\n", - "Processing 859 of 1000 on page 1\n", - "Processing 860 of 1000 on page 1\n", - "Processing 861 of 1000 on page 1\n", - "Processing 862 of 1000 on page 1\n", - "Processing 863 of 1000 on page 1\n", - "Processing 864 of 1000 on page 1\n", - "Processing 865 of 1000 on page 1\n", - "Processing 866 of 1000 on page 1\n", - "Processing 867 of 1000 on page 1\n", - "Processing 868 of 1000 on page 1\n", - "Processing 869 of 1000 on page 1\n", - "Processing 870 of 1000 on page 1\n", - "Processing 871 of 1000 on page 1\n", - "Processing 872 of 1000 on page 1\n", - "Processing 873 of 1000 on page 1\n", - "Processing 874 of 1000 on page 1\n", - "Processing 875 of 1000 on page 1\n", - "Processing 876 of 1000 on page 1\n", - "Processing 877 of 1000 on page 1\n", - "Processing 878 of 1000 on page 1\n", - "Processing 879 of 1000 on page 1\n", - "Processing 880 of 1000 on page 1\n", - "Processing 881 of 1000 on page 1\n", - "Processing 882 of 1000 on page 1\n", - "Processing 883 of 1000 on page 1\n", - "Processing 884 of 1000 on page 1\n", - "Processing 885 of 1000 on page 1\n", - "Processing 886 of 1000 on page 1\n", - "Processing 887 of 1000 on page 1\n", - "Processing 888 of 1000 on page 1\n", - "Processing 889 of 1000 on page 1\n", - "Processing 890 of 1000 on page 1\n", - "Processing 891 of 1000 on page 1\n", - "Processing 892 of 1000 on page 1\n", - "Processing 893 of 1000 on page 1\n", - "Processing 894 of 1000 on page 1\n", - "Processing 895 of 1000 on page 1\n", - "Processing 896 of 1000 on page 1\n", - "Processing 897 of 1000 on page 1\n", - "Processing 898 of 1000 on page 1\n", - "Processing 899 of 1000 on page 1\n", - "Processing 900 of 1000 on page 1\n", - "Processing 901 of 1000 on page 1\n", - "Processing 902 of 1000 on page 1\n", - "Processing 903 of 1000 on page 1\n", - "Processing 904 of 1000 on page 1\n", - "Processing 905 of 1000 on page 1\n", - "Processing 906 of 1000 on page 1\n", - "Processing 907 of 1000 on page 1\n", - "Processing 908 of 1000 on page 1\n", - "Processing 909 of 1000 on page 1\n", - "Processing 910 of 1000 on page 1\n", - "Processing 911 of 1000 on page 1\n", - "Processing 912 of 1000 on page 1\n", - "Processing 913 of 1000 on page 1\n", - "Processing 914 of 1000 on page 1\n", - "Processing 915 of 1000 on page 1\n", - "Processing 916 of 1000 on page 1\n", - "Processing 917 of 1000 on page 1\n", - "Processing 918 of 1000 on page 1\n", - "Processing 919 of 1000 on page 1\n", - "Processing 920 of 1000 on page 1\n", - "Processing 921 of 1000 on page 1\n", - "Processing 922 of 1000 on page 1\n", - "Processing 923 of 1000 on page 1\n", - "Processing 924 of 1000 on page 1\n", - "Processing 925 of 1000 on page 1\n", - "Processing 926 of 1000 on page 1\n", - "Processing 927 of 1000 on page 1\n", - "Processing 928 of 1000 on page 1\n", - "Processing 929 of 1000 on page 1\n", - "Processing 930 of 1000 on page 1\n", - "Processing 931 of 1000 on page 1\n", - "Processing 932 of 1000 on page 1\n", - "Processing 933 of 1000 on page 1\n", - "Processing 934 of 1000 on page 1\n", - "Processing 935 of 1000 on page 1\n", - "Processing 936 of 1000 on page 1\n", - "Processing 937 of 1000 on page 1\n", - "Processing 938 of 1000 on page 1\n", - "Processing 939 of 1000 on page 1\n", - "Processing 940 of 1000 on page 1\n", - "Processing 941 of 1000 on page 1\n", - "Processing 942 of 1000 on page 1\n", - "Processing 943 of 1000 on page 1\n", - "Processing 944 of 1000 on page 1\n", - "Processing 945 of 1000 on page 1\n", - "Processing 946 of 1000 on page 1\n", - "Processing 947 of 1000 on page 1\n", - "Processing 948 of 1000 on page 1\n", - "Processing 949 of 1000 on page 1\n", - "Processing 950 of 1000 on page 1\n", - "Processing 951 of 1000 on page 1\n", - "Processing 952 of 1000 on page 1\n", - "Processing 953 of 1000 on page 1\n", - "Processing 954 of 1000 on page 1\n", - "Processing 955 of 1000 on page 1\n", - "Processing 956 of 1000 on page 1\n", - "Processing 957 of 1000 on page 1\n", - "Processing 958 of 1000 on page 1\n", - "Processing 959 of 1000 on page 1\n", - "Processing 960 of 1000 on page 1\n", - "Processing 961 of 1000 on page 1\n", - "Processing 962 of 1000 on page 1\n", - "Processing 963 of 1000 on page 1\n", - "Processing 964 of 1000 on page 1\n", - "Processing 965 of 1000 on page 1\n", - "Processing 966 of 1000 on page 1\n", - "Processing 967 of 1000 on page 1\n", - "Processing 968 of 1000 on page 1\n", - "Processing 969 of 1000 on page 1\n", - "Processing 970 of 1000 on page 1\n", - "Processing 971 of 1000 on page 1\n", - "Processing 972 of 1000 on page 1\n", - "Processing 973 of 1000 on page 1\n", - "Processing 974 of 1000 on page 1\n", - "Processing 975 of 1000 on page 1\n", - "Processing 976 of 1000 on page 1\n", - "Processing 977 of 1000 on page 1\n", - "Processing 978 of 1000 on page 1\n", - "Processing 979 of 1000 on page 1\n", - "Processing 980 of 1000 on page 1\n", - "Processing 981 of 1000 on page 1\n", - "Processing 982 of 1000 on page 1\n", - "Processing 983 of 1000 on page 1\n", - "Processing 984 of 1000 on page 1\n", - "Processing 985 of 1000 on page 1\n", - "Processing 986 of 1000 on page 1\n", - "Processing 987 of 1000 on page 1\n", - "Processing 988 of 1000 on page 1\n", - "Processing 989 of 1000 on page 1\n", - "Processing 990 of 1000 on page 1\n", - "Processing 991 of 1000 on page 1\n", - "Processing 992 of 1000 on page 1\n", - "Processing 993 of 1000 on page 1\n", - "Processing 994 of 1000 on page 1\n", - "Processing 995 of 1000 on page 1\n", - "Processing 996 of 1000 on page 1\n", - "Processing 997 of 1000 on page 1\n", - "Processing 998 of 1000 on page 1\n", - "Processing 999 of 1000 on page 1\n", - "Processing 1000 of 1000 on page 1\n", - "Processing 1001 of 1000 on page 2\n", - "Processing 1002 of 1000 on page 2\n", - "Processing 1003 of 1000 on page 2\n", - "Processing 1004 of 1000 on page 2\n", - "Processing 1005 of 1000 on page 2\n", - "Processing 1006 of 1000 on page 2\n", - "Processing 1007 of 1000 on page 2\n", - "Processing 1008 of 1000 on page 2\n", - "Processing 1009 of 1000 on page 2\n", - "Processing 1010 of 1000 on page 2\n", - "Processing 1011 of 1000 on page 2\n", - "Processing 1012 of 1000 on page 2\n", - "Processing 1013 of 1000 on page 2\n", - "Processing 1014 of 1000 on page 2\n", - "Processing 1015 of 1000 on page 2\n", - "Processing 1016 of 1000 on page 2\n", - "Processing 1017 of 1000 on page 2\n", - "Processing 1018 of 1000 on page 2\n", - "Processing 1019 of 1000 on page 2\n", - "Processing 1020 of 1000 on page 2\n", - "Processing 1021 of 1000 on page 2\n", - "Processing 1022 of 1000 on page 2\n", - "Processing 1023 of 1000 on page 2\n", - "Processing 1024 of 1000 on page 2\n", - "Processing 1025 of 1000 on page 2\n", - "Processing 1026 of 1000 on page 2\n", - "Processing 1027 of 1000 on page 2\n", - "Processing 1028 of 1000 on page 2\n", - "Processing 1029 of 1000 on page 2\n", - "Processing 1030 of 1000 on page 2\n", - "Processing 1031 of 1000 on page 2\n", - "Processing 1032 of 1000 on page 2\n", - "Processing 1033 of 1000 on page 2\n", - "Processing 1034 of 1000 on page 2\n", - "Processing 1035 of 1000 on page 2\n", - "Processing 1036 of 1000 on page 2\n", - "Processing 1037 of 1000 on page 2\n", - "Processing 1038 of 1000 on page 2\n", - "Processing 1039 of 1000 on page 2\n", - "Processing 1040 of 1000 on page 2\n", - "Processing 1041 of 1000 on page 2\n", - "Processing 1042 of 1000 on page 2\n", - "Processing 1043 of 1000 on page 2\n", - "Processing 1044 of 1000 on page 2\n", - "Processing 1045 of 1000 on page 2\n", - "Processing 1046 of 1000 on page 2\n", - "Processing 1047 of 1000 on page 2\n", - "Processing 1048 of 1000 on page 2\n", - "Processing 1049 of 1000 on page 2\n", - "Processing 1050 of 1000 on page 2\n", - "Processing 1051 of 1000 on page 2\n", - "Processing 1052 of 1000 on page 2\n", - "Processing 1053 of 1000 on page 2\n", - "Processing 1054 of 1000 on page 2\n", - "Processing 1055 of 1000 on page 2\n", - "Processing 1056 of 1000 on page 2\n", - "Processing 1057 of 1000 on page 2\n", - "Processing 1058 of 1000 on page 2\n", - "Processing 1059 of 1000 on page 2\n", - "Processing 1060 of 1000 on page 2\n", - "Processing 1061 of 1000 on page 2\n", - "Processing 1062 of 1000 on page 2\n", - "Processing 1063 of 1000 on page 2\n", - "Processing 1064 of 1000 on page 2\n", - "Processing 1065 of 1000 on page 2\n", - "Processing 1066 of 1000 on page 2\n", - "Processing 1067 of 1000 on page 2\n", - "Processing 1068 of 1000 on page 2\n", - "Processing 1069 of 1000 on page 2\n", - "Processing 1070 of 1000 on page 2\n", - "Processing 1071 of 1000 on page 2\n", - "Processing 1072 of 1000 on page 2\n", - "Processing 1073 of 1000 on page 2\n", - "Processing 1074 of 1000 on page 2\n", - "Processing 1075 of 1000 on page 2\n", - "Processing 1076 of 1000 on page 2\n", - "Processing 1077 of 1000 on page 2\n", - "Processing 1078 of 1000 on page 2\n", - "Processing 1079 of 1000 on page 2\n", - "Processing 1080 of 1000 on page 2\n", - "Processing 1081 of 1000 on page 2\n", - "Processing 1082 of 1000 on page 2\n", - "Processing 1083 of 1000 on page 2\n", - "Processing 1084 of 1000 on page 2\n", - "Processing 1085 of 1000 on page 2\n", - "Processing 1086 of 1000 on page 2\n", - "Processing 1087 of 1000 on page 2\n", - "Processing 1088 of 1000 on page 2\n", - "Processing 1089 of 1000 on page 2\n", - "Processing 1090 of 1000 on page 2\n", - "Processing 1091 of 1000 on page 2\n", - "Processing 1092 of 1000 on page 2\n", - "Processing 1093 of 1000 on page 2\n", - "Processing 1094 of 1000 on page 2\n", - "Processing 1095 of 1000 on page 2\n", - "Processing 1096 of 1000 on page 2\n", - "Processing 1097 of 1000 on page 2\n", - "Processing 1098 of 1000 on page 2\n", - "Processing 1099 of 1000 on page 2\n", - "Processing 1100 of 1000 on page 2\n", - "Processing 1101 of 1000 on page 2\n", - "Processing 1102 of 1000 on page 2\n", - "Processing 1103 of 1000 on page 2\n", - "Processing 1104 of 1000 on page 2\n", - "Processing 1105 of 1000 on page 2\n", - "Processing 1106 of 1000 on page 2\n", - "Processing 1107 of 1000 on page 2\n", - "Processing 1108 of 1000 on page 2\n", - "Processing 1109 of 1000 on page 2\n", - "Processing 1110 of 1000 on page 2\n", - "Processing 1111 of 1000 on page 2\n", - "Processing 1112 of 1000 on page 2\n", - "Processing 1113 of 1000 on page 2\n", - "Processing 1114 of 1000 on page 2\n", - "Processing 1115 of 1000 on page 2\n", - "Processing 1116 of 1000 on page 2\n", - "Processing 1117 of 1000 on page 2\n", - "Processing 1118 of 1000 on page 2\n", - "Processing 1119 of 1000 on page 2\n", - "Processing 1120 of 1000 on page 2\n", - "Processing 1121 of 1000 on page 2\n", - "Processing 1122 of 1000 on page 2\n", - "Processing 1123 of 1000 on page 2\n", - "Processing 1124 of 1000 on page 2\n", - "Processing 1125 of 1000 on page 2\n", - "Processing 1126 of 1000 on page 2\n", - "Processing 1127 of 1000 on page 2\n", - "Processing 1128 of 1000 on page 2\n", - "Processing 1129 of 1000 on page 2\n", - "Processing 1130 of 1000 on page 2\n", - "Processing 1131 of 1000 on page 2\n", - "Processing 1132 of 1000 on page 2\n", - "Processing 1133 of 1000 on page 2\n", - "Processing 1134 of 1000 on page 2\n", - "Processing 1135 of 1000 on page 2\n", - "Processing 1136 of 1000 on page 2\n", - "Processing 1137 of 1000 on page 2\n", - "Processing 1138 of 1000 on page 2\n", - "Processing 1139 of 1000 on page 2\n", - "Processing 1140 of 1000 on page 2\n", - "Processing 1141 of 1000 on page 2\n", - "Processing 1142 of 1000 on page 2\n", - "Processing 1143 of 1000 on page 2\n", - "Processing 1144 of 1000 on page 2\n", - "Processing 1145 of 1000 on page 2\n", - "Processing 1146 of 1000 on page 2\n", - "Processing 1147 of 1000 on page 2\n", - "Processing 1148 of 1000 on page 2\n", - "Processing 1149 of 1000 on page 2\n", - "Processing 1150 of 1000 on page 2\n", - "Processing 1151 of 1000 on page 2\n", - "Processing 1152 of 1000 on page 2\n", - "Processing 1153 of 1000 on page 2\n", - "Processing 1154 of 1000 on page 2\n", - "Processing 1155 of 1000 on page 2\n", - "Processing 1156 of 1000 on page 2\n", - "Processing 1157 of 1000 on page 2\n", - "Processing 1158 of 1000 on page 2\n", - "Processing 1159 of 1000 on page 2\n", - "Processing 1160 of 1000 on page 2\n", - "Processing 1161 of 1000 on page 2\n", - "Processing 1162 of 1000 on page 2\n", - "Processing 1163 of 1000 on page 2\n", - "Processing 1164 of 1000 on page 2\n", - "Processing 1165 of 1000 on page 2\n", - "Processing 1166 of 1000 on page 2\n", - "Processing 1167 of 1000 on page 2\n", - "Processing 1168 of 1000 on page 2\n", - "Processing 1169 of 1000 on page 2\n", - "Processing 1170 of 1000 on page 2\n", - "Processing 1171 of 1000 on page 2\n", - "Processing 1172 of 1000 on page 2\n", - "Processing 1173 of 1000 on page 2\n", - "Processing 1174 of 1000 on page 2\n", - "Processing 1175 of 1000 on page 2\n", - "Processing 1176 of 1000 on page 2\n", - "Processing 1177 of 1000 on page 2\n", - "Processing 1178 of 1000 on page 2\n", - "Processing 1179 of 1000 on page 2\n", - "Processing 1180 of 1000 on page 2\n", - "Processing 1181 of 1000 on page 2\n", - "Processing 1182 of 1000 on page 2\n", - "Processing 1183 of 1000 on page 2\n", - "Processing 1184 of 1000 on page 2\n", - "Processing 1185 of 1000 on page 2\n", - "Processing 1186 of 1000 on page 2\n", - "Processing 1187 of 1000 on page 2\n", - "Processing 1188 of 1000 on page 2\n", - "Processing 1189 of 1000 on page 2\n", - "Processing 1190 of 1000 on page 2\n", - "Processing 1191 of 1000 on page 2\n", - "Processing 1192 of 1000 on page 2\n", - "Processing 1193 of 1000 on page 2\n", - "Processing 1194 of 1000 on page 2\n", - "Processing 1195 of 1000 on page 2\n", - "Processing 1196 of 1000 on page 2\n", - "Processing 1197 of 1000 on page 2\n", - "Processing 1198 of 1000 on page 2\n", - "Processing 1199 of 1000 on page 2\n", - "Processing 1200 of 1000 on page 2\n", - "Processing 1201 of 1000 on page 2\n", - "Processing 1202 of 1000 on page 2\n", - "Processing 1203 of 1000 on page 2\n", - "Processing 1204 of 1000 on page 2\n", - "Processing 1205 of 1000 on page 2\n", - "Processing 1206 of 1000 on page 2\n", - "Processing 1207 of 1000 on page 2\n", - "Processing 1208 of 1000 on page 2\n", - "Processing 1209 of 1000 on page 2\n", - "Processing 1210 of 1000 on page 2\n", - "Processing 1211 of 1000 on page 2\n", - "Processing 1212 of 1000 on page 2\n", - "Processing 1213 of 1000 on page 2\n", - "Processing 1214 of 1000 on page 2\n", - "Processing 1215 of 1000 on page 2\n", - "Processing 1216 of 1000 on page 2\n", - "Processing 1217 of 1000 on page 2\n", - "Processing 1218 of 1000 on page 2\n", - "Processing 1219 of 1000 on page 2\n", - "Processing 1220 of 1000 on page 2\n", - "Processing 1221 of 1000 on page 2\n", - "Processing 1222 of 1000 on page 2\n", - "Processing 1223 of 1000 on page 2\n", - "Processing 1224 of 1000 on page 2\n", - "Processing 1225 of 1000 on page 2\n", - "Processing 1226 of 1000 on page 2\n", - "Processing 1227 of 1000 on page 2\n", - "Processing 1228 of 1000 on page 2\n", - "Processing 1229 of 1000 on page 2\n", - "Processing 1230 of 1000 on page 2\n", - "Processing 1231 of 1000 on page 2\n", - "Processing 1232 of 1000 on page 2\n", - "Processing 1233 of 1000 on page 2\n", - "Processing 1234 of 1000 on page 2\n", - "Processing 1235 of 1000 on page 2\n", - "Processing 1236 of 1000 on page 2\n", - "Processing 1237 of 1000 on page 2\n", - "Processing 1238 of 1000 on page 2\n", - "Processing 1239 of 1000 on page 2\n", - "Processing 1240 of 1000 on page 2\n", - "Processing 1241 of 1000 on page 2\n", - "Processing 1242 of 1000 on page 2\n", - "Processing 1243 of 1000 on page 2\n", - "Processing 1244 of 1000 on page 2\n", - "Processing 1245 of 1000 on page 2\n", - "Processing 1246 of 1000 on page 2\n", - "Processing 1247 of 1000 on page 2\n", - "Processing 1248 of 1000 on page 2\n", - "Processing 1249 of 1000 on page 2\n", - "Processing 1250 of 1000 on page 2\n", - "Processing 1251 of 1000 on page 2\n", - "Processing 1252 of 1000 on page 2\n", - "Processing 1253 of 1000 on page 2\n", - "Processing 1254 of 1000 on page 2\n", - "Processing 1255 of 1000 on page 2\n", - "Processing 1256 of 1000 on page 2\n", - "Processing 1257 of 1000 on page 2\n", - "Processing 1258 of 1000 on page 2\n", - "Processing 1259 of 1000 on page 2\n", - "Processing 1260 of 1000 on page 2\n", - "Processing 1261 of 1000 on page 2\n", - "Processing 1262 of 1000 on page 2\n", - "Processing 1263 of 1000 on page 2\n", - "Processing 1264 of 1000 on page 2\n", - "Processing 1265 of 1000 on page 2\n", - "Processing 1266 of 1000 on page 2\n", - "Processing 1267 of 1000 on page 2\n", - "Processing 1268 of 1000 on page 2\n", - "Processing 1269 of 1000 on page 2\n", - "Processing 1270 of 1000 on page 2\n", - "Processing 1271 of 1000 on page 2\n", - "Processing 1272 of 1000 on page 2\n", - "Processing 1273 of 1000 on page 2\n", - "Processing 1274 of 1000 on page 2\n", - "Processing 1275 of 1000 on page 2\n", - "Processing 1276 of 1000 on page 2\n", - "Processing 1277 of 1000 on page 2\n", - "Processing 1278 of 1000 on page 2\n", - "Processing 1279 of 1000 on page 2\n", - "Processing 1280 of 1000 on page 2\n", - "Processing 1281 of 1000 on page 2\n", - "Processing 1282 of 1000 on page 2\n", - "Processing 1283 of 1000 on page 2\n", - "Processing 1284 of 1000 on page 2\n", - "Processing 1285 of 1000 on page 2\n", - "Processing 1286 of 1000 on page 2\n", - "Processing 1287 of 1000 on page 2\n", - "Processing 1288 of 1000 on page 2\n", - "Processing 1289 of 1000 on page 2\n", - "Processing 1290 of 1000 on page 2\n", - "Processing 1291 of 1000 on page 2\n", - "Processing 1292 of 1000 on page 2\n", - "Processing 1293 of 1000 on page 2\n", - "Processing 1294 of 1000 on page 2\n", - "Processing 1295 of 1000 on page 2\n", - "Processing 1296 of 1000 on page 2\n", - "Processing 1297 of 1000 on page 2\n", - "Processing 1298 of 1000 on page 2\n", - "Processing 1299 of 1000 on page 2\n", - "Processing 1300 of 1000 on page 2\n", - "Processing 1301 of 1000 on page 2\n", - "Processing 1302 of 1000 on page 2\n", - "Processing 1303 of 1000 on page 2\n", - "Processing 1304 of 1000 on page 2\n", - "Processing 1305 of 1000 on page 2\n", - "Processing 1306 of 1000 on page 2\n", - "Processing 1307 of 1000 on page 2\n", - "Processing 1308 of 1000 on page 2\n", - "Processing 1309 of 1000 on page 2\n", - "Processing 1310 of 1000 on page 2\n", - "Processing 1311 of 1000 on page 2\n", - "Processing 1312 of 1000 on page 2\n", - "Processing 1313 of 1000 on page 2\n", - "Processing 1314 of 1000 on page 2\n", - "Processing 1315 of 1000 on page 2\n", - "Processing 1316 of 1000 on page 2\n", - "Processing 1317 of 1000 on page 2\n", - "Processing 1318 of 1000 on page 2\n", - "Processing 1319 of 1000 on page 2\n", - "Processing 1320 of 1000 on page 2\n", - "Processing 1321 of 1000 on page 2\n", - "Processing 1322 of 1000 on page 2\n", - "Processing 1323 of 1000 on page 2\n", - "Processing 1324 of 1000 on page 2\n", - "Processing 1325 of 1000 on page 2\n", - "Processing 1326 of 1000 on page 2\n", - "Processing 1327 of 1000 on page 2\n", - "Processing 1328 of 1000 on page 2\n", - "Processing 1329 of 1000 on page 2\n", - "Processing 1330 of 1000 on page 2\n", - "Processing 1331 of 1000 on page 2\n", - "Processing 1332 of 1000 on page 2\n", - "Processing 1333 of 1000 on page 2\n", - "Processing 1334 of 1000 on page 2\n", - "Processing 1335 of 1000 on page 2\n", - "Processing 1336 of 1000 on page 2\n", - "Processing 1337 of 1000 on page 2\n", - "Processing 1338 of 1000 on page 2\n", - "Processing 1339 of 1000 on page 2\n", - "Processing 1340 of 1000 on page 2\n", - "Processing 1341 of 1000 on page 2\n", - "Processing 1342 of 1000 on page 2\n", - "Processing 1343 of 1000 on page 2\n", - "Processing 1344 of 1000 on page 2\n", - "Processing 1345 of 1000 on page 2\n", - "Processing 1346 of 1000 on page 2\n", - "Processing 1347 of 1000 on page 2\n", - "Processing 1348 of 1000 on page 2\n", - "Processing 1349 of 1000 on page 2\n", - "Processing 1350 of 1000 on page 2\n", - "Processing 1351 of 1000 on page 2\n", - "Processing 1352 of 1000 on page 2\n", - "Processing 1353 of 1000 on page 2\n", - "Processing 1354 of 1000 on page 2\n", - "Processing 1355 of 1000 on page 2\n", - "Processing 1356 of 1000 on page 2\n", - "Processing 1357 of 1000 on page 2\n", - "Processing 1358 of 1000 on page 2\n", - "Processing 1359 of 1000 on page 2\n", - "Processing 1360 of 1000 on page 2\n", - "Processing 1361 of 1000 on page 2\n", - "Processing 1362 of 1000 on page 2\n", - "Processing 1363 of 1000 on page 2\n", - "Processing 1364 of 1000 on page 2\n", - "Processing 1365 of 1000 on page 2\n", - "Processing 1366 of 1000 on page 2\n", - "Processing 1367 of 1000 on page 2\n", - "Processing 1368 of 1000 on page 2\n", - "Processing 1369 of 1000 on page 2\n", - "Processing 1370 of 1000 on page 2\n", - "Processing 1371 of 1000 on page 2\n", - "Processing 1372 of 1000 on page 2\n", - "Processing 1373 of 1000 on page 2\n", - "Processing 1374 of 1000 on page 2\n", - "Processing 1375 of 1000 on page 2\n", - "Processing 1376 of 1000 on page 2\n", - "Processing 1377 of 1000 on page 2\n", - "Processing 1378 of 1000 on page 2\n", - "Processing 1379 of 1000 on page 2\n", - "Processing 1380 of 1000 on page 2\n", - "Processing 1381 of 1000 on page 2\n", - "Processing 1382 of 1000 on page 2\n", - "Processing 1383 of 1000 on page 2\n", - "Processing 1384 of 1000 on page 2\n", - "Processing 1385 of 1000 on page 2\n", - "Processing 1386 of 1000 on page 2\n", - "Processing 1387 of 1000 on page 2\n", - "Processing 1388 of 1000 on page 2\n", - "Processing 1389 of 1000 on page 2\n", - "Processing 1390 of 1000 on page 2\n", - "Processing 1391 of 1000 on page 2\n", - "Processing 1392 of 1000 on page 2\n", - "Processing 1393 of 1000 on page 2\n", - "Processing 1394 of 1000 on page 2\n", - "Processing 1395 of 1000 on page 2\n", - "Processing 1396 of 1000 on page 2\n", - "Processing 1397 of 1000 on page 2\n", - "Processing 1398 of 1000 on page 2\n", - "Processing 1399 of 1000 on page 2\n", - "Processing 1400 of 1000 on page 2\n", - "Processing 1401 of 1000 on page 2\n", - "Processing 1402 of 1000 on page 2\n", - "Processing 1403 of 1000 on page 2\n", - "Processing 1404 of 1000 on page 2\n", - "Processing 1405 of 1000 on page 2\n", - "Processing 1406 of 1000 on page 2\n", - "Processing 1407 of 1000 on page 2\n", - "Processing 1408 of 1000 on page 2\n", - "Processing 1409 of 1000 on page 2\n", - "Processing 1410 of 1000 on page 2\n", - "Processing 1411 of 1000 on page 2\n", - "Processing 1412 of 1000 on page 2\n", - "Processing 1413 of 1000 on page 2\n", - "Processing 1414 of 1000 on page 2\n", - "Processing 1415 of 1000 on page 2\n", - "Processing 1416 of 1000 on page 2\n", - "Processing 1417 of 1000 on page 2\n", - "Processing 1418 of 1000 on page 2\n", - "Processing 1419 of 1000 on page 2\n", - "Processing 1420 of 1000 on page 2\n", - "Processing 1421 of 1000 on page 2\n", - "Processing 1422 of 1000 on page 2\n", - "Processing 1423 of 1000 on page 2\n", - "Processing 1424 of 1000 on page 2\n", - "Processing 1425 of 1000 on page 2\n", - "Processing 1426 of 1000 on page 2\n", - "Processing 1427 of 1000 on page 2\n", - "Processing 1428 of 1000 on page 2\n", - "Processing 1429 of 1000 on page 2\n", - "Processing 1430 of 1000 on page 2\n", - "Processing 1431 of 1000 on page 2\n", - "Processing 1432 of 1000 on page 2\n", - "Processing 1433 of 1000 on page 2\n", - "Processing 1434 of 1000 on page 2\n", - "Processing 1435 of 1000 on page 2\n", - "Processing 1436 of 1000 on page 2\n", - "Processing 1437 of 1000 on page 2\n", - "Processing 1438 of 1000 on page 2\n", - "Processing 1439 of 1000 on page 2\n", - "Processing 1440 of 1000 on page 2\n", - "Processing 1441 of 1000 on page 2\n", - "Processing 1442 of 1000 on page 2\n", - "Processing 1443 of 1000 on page 2\n", - "Processing 1444 of 1000 on page 2\n", - "Processing 1445 of 1000 on page 2\n", - "Processing 1446 of 1000 on page 2\n", - "Processing 1447 of 1000 on page 2\n", - "Processing 1448 of 1000 on page 2\n", - "Processing 1449 of 1000 on page 2\n", - "Processing 1450 of 1000 on page 2\n", - "Processing 1451 of 1000 on page 2\n", - "Processing 1452 of 1000 on page 2\n", - "Processing 1453 of 1000 on page 2\n", - "Processing 1454 of 1000 on page 2\n", - "Processing 1455 of 1000 on page 2\n", - "Processing 1456 of 1000 on page 2\n", - "Processing 1457 of 1000 on page 2\n", - "Processing 1458 of 1000 on page 2\n", - "Processing 1459 of 1000 on page 2\n", - "Processing 1460 of 1000 on page 2\n", - "Processing 1461 of 1000 on page 2\n", - "Processing 1462 of 1000 on page 2\n", - "Processing 1463 of 1000 on page 2\n", - "Processing 1464 of 1000 on page 2\n", - "Processing 1465 of 1000 on page 2\n", - "Processing 1466 of 1000 on page 2\n", - "Processing 1467 of 1000 on page 2\n", - "Processing 1468 of 1000 on page 2\n", - "Processing 1469 of 1000 on page 2\n", - "Processing 1470 of 1000 on page 2\n", - "Processing 1471 of 1000 on page 2\n", - "Processing 1472 of 1000 on page 2\n", - "Processing 1473 of 1000 on page 2\n", - "Processing 1474 of 1000 on page 2\n", - "Processing 1475 of 1000 on page 2\n", - "Processing 1476 of 1000 on page 2\n", - "Processing 1477 of 1000 on page 2\n", - "Processing 1478 of 1000 on page 2\n", - "Processing 1479 of 1000 on page 2\n", - "Processing 1480 of 1000 on page 2\n", - "Processing 1481 of 1000 on page 2\n", - "Processing 1482 of 1000 on page 2\n", - "Processing 1483 of 1000 on page 2\n", - "Processing 1484 of 1000 on page 2\n", - "Processing 1485 of 1000 on page 2\n", - "Processing 1486 of 1000 on page 2\n", - "Processing 1487 of 1000 on page 2\n", - "Processing 1488 of 1000 on page 2\n", - "Processing 1489 of 1000 on page 2\n", - "Processing 1490 of 1000 on page 2\n", - "Processing 1491 of 1000 on page 2\n", - "Processing 1492 of 1000 on page 2\n", - "Processing 1493 of 1000 on page 2\n", - "Processing 1494 of 1000 on page 2\n", - "Processing 1495 of 1000 on page 2\n", - "Processing 1496 of 1000 on page 2\n", - "Processing 1497 of 1000 on page 2\n", - "Processing 1498 of 1000 on page 2\n", - "Processing 1499 of 1000 on page 2\n", - "Processing 1500 of 1000 on page 2\n", - "Processing 1501 of 1000 on page 2\n", - "Processing 1502 of 1000 on page 2\n", - "Processing 1503 of 1000 on page 2\n", - "Processing 1504 of 1000 on page 2\n", - "Processing 1505 of 1000 on page 2\n", - "Processing 1506 of 1000 on page 2\n", - "Processing 1507 of 1000 on page 2\n", - "Processing 1508 of 1000 on page 2\n", - "Processing 1509 of 1000 on page 2\n", - "Processing 1510 of 1000 on page 2\n", - "Processing 1511 of 1000 on page 2\n", - "Processing 1512 of 1000 on page 2\n", - "Processing 1513 of 1000 on page 2\n", - "Processing 1514 of 1000 on page 2\n", - "Processing 1515 of 1000 on page 2\n", - "Processing 1516 of 1000 on page 2\n", - "Processing 1517 of 1000 on page 2\n", - "Processing 1518 of 1000 on page 2\n", - "Processing 1519 of 1000 on page 2\n", - "Processing 1520 of 1000 on page 2\n", - "Processing 1521 of 1000 on page 2\n", - "Processing 1522 of 1000 on page 2\n", - "Processing 1523 of 1000 on page 2\n", - "Processing 1524 of 1000 on page 2\n", - "Processing 1525 of 1000 on page 2\n", - "Processing 1526 of 1000 on page 2\n", - "Processing 1527 of 1000 on page 2\n", - "Processing 1528 of 1000 on page 2\n", - "Processing 1529 of 1000 on page 2\n", - "Processing 1530 of 1000 on page 2\n", - "Processing 1531 of 1000 on page 2\n", - "Processing 1532 of 1000 on page 2\n", - "Processing 1533 of 1000 on page 2\n", - "Processing 1534 of 1000 on page 2\n", - "Processing 1535 of 1000 on page 2\n", - "Processing 1536 of 1000 on page 2\n", - "Processing 1537 of 1000 on page 2\n", - "Processing 1538 of 1000 on page 2\n", - "Processing 1539 of 1000 on page 2\n", - "Processing 1540 of 1000 on page 2\n", - "Processing 1541 of 1000 on page 2\n", - "Processing 1542 of 1000 on page 2\n", - "Processing 1543 of 1000 on page 2\n", - "Processing 1544 of 1000 on page 2\n", - "Processing 1545 of 1000 on page 2\n", - "Processing 1546 of 1000 on page 2\n", - "Processing 1547 of 1000 on page 2\n", - "Processing 1548 of 1000 on page 2\n", - "Processing 1549 of 1000 on page 2\n", - "Processing 1550 of 1000 on page 2\n", - "Processing 1551 of 1000 on page 2\n", - "Processing 1552 of 1000 on page 2\n", - "Processing 1553 of 1000 on page 2\n", - "Processing 1554 of 1000 on page 2\n", - "Processing 1555 of 1000 on page 2\n", - "Processing 1556 of 1000 on page 2\n", - "Processing 1557 of 1000 on page 2\n", - "Processing 1558 of 1000 on page 2\n", - "Processing 1559 of 1000 on page 2\n", - "Processing 1560 of 1000 on page 2\n", - "Processing 1561 of 1000 on page 2\n", - "Processing 1562 of 1000 on page 2\n", - "Processing 1563 of 1000 on page 2\n", - "Processing 1564 of 1000 on page 2\n", - "Processing 1565 of 1000 on page 2\n", - "Processing 1566 of 1000 on page 2\n", - "Processing 1567 of 1000 on page 2\n", - "Processing 1568 of 1000 on page 2\n", - "Processing 1569 of 1000 on page 2\n", - "Processing 1570 of 1000 on page 2\n", - "Processing 1571 of 1000 on page 2\n", - "Processing 1572 of 1000 on page 2\n", - "Processing 1573 of 1000 on page 2\n", - "Processing 1574 of 1000 on page 2\n", - "Processing 1575 of 1000 on page 2\n", - "Processing 1576 of 1000 on page 2\n", - "Processing 1577 of 1000 on page 2\n", - "Processing 1578 of 1000 on page 2\n", - "Processing 1579 of 1000 on page 2\n", - "Processing 1580 of 1000 on page 2\n", - "Processing 1581 of 1000 on page 2\n", - "Processing 1582 of 1000 on page 2\n", - "Processing 1583 of 1000 on page 2\n", - "Processing 1584 of 1000 on page 2\n", - "Processing 1585 of 1000 on page 2\n", - "Processing 1586 of 1000 on page 2\n", - "Processing 1587 of 1000 on page 2\n", - "Processing 1588 of 1000 on page 2\n", - "Processing 1589 of 1000 on page 2\n", - "Processing 1590 of 1000 on page 2\n", - "Processing 1591 of 1000 on page 2\n", - "Processing 1592 of 1000 on page 2\n", - "Processing 1593 of 1000 on page 2\n", - "Processing 1594 of 1000 on page 2\n", - "Processing 1595 of 1000 on page 2\n", - "Processing 1596 of 1000 on page 2\n", - "Processing 1597 of 1000 on page 2\n", - "Processing 1598 of 1000 on page 2\n", - "Processing 1599 of 1000 on page 2\n", - "Processing 1600 of 1000 on page 2\n", - "Processing 1601 of 1000 on page 2\n", - "Processing 1602 of 1000 on page 2\n", - "Processing 1603 of 1000 on page 2\n", - "Processing 1604 of 1000 on page 2\n", - "Processing 1605 of 1000 on page 2\n", - "Processing 1606 of 1000 on page 2\n", - "Processing 1607 of 1000 on page 2\n", - "Processing 1608 of 1000 on page 2\n", - "Processing 1609 of 1000 on page 2\n", - "Processing 1610 of 1000 on page 2\n", - "Processing 1611 of 1000 on page 2\n", - "Processing 1612 of 1000 on page 2\n", - "Processing 1613 of 1000 on page 2\n", - "Processing 1614 of 1000 on page 2\n", - "Processing 1615 of 1000 on page 2\n", - "Processing 1616 of 1000 on page 2\n", - "Processing 1617 of 1000 on page 2\n", - "Processing 1618 of 1000 on page 2\n", - "Processing 1619 of 1000 on page 2\n", - "Processing 1620 of 1000 on page 2\n", - "Processing 1621 of 1000 on page 2\n", - "Processing 1622 of 1000 on page 2\n", - "Processing 1623 of 1000 on page 2\n", - "Processing 1624 of 1000 on page 2\n", - "Processing 1625 of 1000 on page 2\n", - "Processing 1626 of 1000 on page 2\n", - "Processing 1627 of 1000 on page 2\n", - "Processing 1628 of 1000 on page 2\n", - "Processing 1629 of 1000 on page 2\n", - "Processing 1630 of 1000 on page 2\n", - "Processing 1631 of 1000 on page 2\n", - "Processing 1632 of 1000 on page 2\n", - "Processing 1633 of 1000 on page 2\n", - "Processing 1634 of 1000 on page 2\n", - "Processing 1635 of 1000 on page 2\n", - "Processing 1636 of 1000 on page 2\n", - "Processing 1637 of 1000 on page 2\n", - "Processing 1638 of 1000 on page 2\n", - "Processing 1639 of 1000 on page 2\n", - "Processing 1640 of 1000 on page 2\n", - "Processing 1641 of 1000 on page 2\n", - "Processing 1642 of 1000 on page 2\n", - "Processing 1643 of 1000 on page 2\n", - "Processing 1644 of 1000 on page 2\n", - "Processing 1645 of 1000 on page 2\n", - "Processing 1646 of 1000 on page 2\n", - "Processing 1647 of 1000 on page 2\n", - "Processing 1648 of 1000 on page 2\n", - "Processing 1649 of 1000 on page 2\n", - "Processing 1650 of 1000 on page 2\n", - "Processing 1651 of 1000 on page 2\n", - "Processing 1652 of 1000 on page 2\n", - "Processing 1653 of 1000 on page 2\n", - "Processing 1654 of 1000 on page 2\n", - "Processing 1655 of 1000 on page 2\n", - "Processing 1656 of 1000 on page 2\n", - "Processing 1657 of 1000 on page 2\n", - "Processing 1658 of 1000 on page 2\n", - "Processing 1659 of 1000 on page 2\n", - "Processing 1660 of 1000 on page 2\n", - "Processing 1661 of 1000 on page 2\n", - "Processing 1662 of 1000 on page 2\n", - "Processing 1663 of 1000 on page 2\n", - "Processing 1664 of 1000 on page 2\n", - "Processing 1665 of 1000 on page 2\n", - "Processing 1666 of 1000 on page 2\n", - "Processing 1667 of 1000 on page 2\n", - "Processing 1668 of 1000 on page 2\n", - "Processing 1669 of 1000 on page 2\n", - "Processing 1670 of 1000 on page 2\n", - "Processing 1671 of 1000 on page 2\n", - "Processing 1672 of 1000 on page 2\n", - "Processing 1673 of 1000 on page 2\n", - "Processing 1674 of 1000 on page 2\n", - "Processing 1675 of 1000 on page 2\n", - "Processing 1676 of 1000 on page 2\n", - "Processing 1677 of 1000 on page 2\n", - "Processing 1678 of 1000 on page 2\n", - "Processing 1679 of 1000 on page 2\n", - "Processing 1680 of 1000 on page 2\n", - "Processing 1681 of 1000 on page 2\n", - "Processing 1682 of 1000 on page 2\n", - "Processing 1683 of 1000 on page 2\n", - "Processing 1684 of 1000 on page 2\n", - "Processing 1685 of 1000 on page 2\n", - "Processing 1686 of 1000 on page 2\n", - "Processing 1687 of 1000 on page 2\n", - "Processing 1688 of 1000 on page 2\n", - "Processing 1689 of 1000 on page 2\n", - "Processing 1690 of 1000 on page 2\n", - "Processing 1691 of 1000 on page 2\n", - "Processing 1692 of 1000 on page 2\n", - "Processing 1693 of 1000 on page 2\n", - "Processing 1694 of 1000 on page 2\n", - "Processing 1695 of 1000 on page 2\n", - "Processing 1696 of 1000 on page 2\n", - "Processing 1697 of 1000 on page 2\n", - "Processing 1698 of 1000 on page 2\n", - "Processing 1699 of 1000 on page 2\n", - "Processing 1700 of 1000 on page 2\n", - "Processing 1701 of 1000 on page 2\n", - "Processing 1702 of 1000 on page 2\n", - "Processing 1703 of 1000 on page 2\n", - "Processing 1704 of 1000 on page 2\n", - "Processing 1705 of 1000 on page 2\n", - "Processing 1706 of 1000 on page 2\n", - "Processing 1707 of 1000 on page 2\n", - "Processing 1708 of 1000 on page 2\n", - "Processing 1709 of 1000 on page 2\n", - "Processing 1710 of 1000 on page 2\n", - "Processing 1711 of 1000 on page 2\n", - "Processing 1712 of 1000 on page 2\n", - "Processing 1713 of 1000 on page 2\n", - "Processing 1714 of 1000 on page 2\n", - "Processing 1715 of 1000 on page 2\n", - "Processing 1716 of 1000 on page 2\n", - "Processing 1717 of 1000 on page 2\n", - "Processing 1718 of 1000 on page 2\n", - "Processing 1719 of 1000 on page 2\n", - "Processing 1720 of 1000 on page 2\n", - "Processing 1721 of 1000 on page 2\n", - "Processing 1722 of 1000 on page 2\n", - "Processing 1723 of 1000 on page 2\n", - "Processing 1724 of 1000 on page 2\n", - "Processing 1725 of 1000 on page 2\n", - "Processing 1726 of 1000 on page 2\n", - "Processing 1727 of 1000 on page 2\n", - "Processing 1728 of 1000 on page 2\n", - "Processing 1729 of 1000 on page 2\n", - "Processing 1730 of 1000 on page 2\n", - "Processing 1731 of 1000 on page 2\n", - "Processing 1732 of 1000 on page 2\n", - "Processing 1733 of 1000 on page 2\n", - "Processing 1734 of 1000 on page 2\n", - "Processing 1735 of 1000 on page 2\n", - "Processing 1736 of 1000 on page 2\n", - "Processing 1737 of 1000 on page 2\n", - "Processing 1738 of 1000 on page 2\n", - "Processing 1739 of 1000 on page 2\n", - "Processing 1740 of 1000 on page 2\n", - "Processing 1741 of 1000 on page 2\n", - "Processing 1742 of 1000 on page 2\n", - "Processing 1743 of 1000 on page 2\n", - "Processing 1744 of 1000 on page 2\n", - "Processing 1745 of 1000 on page 2\n", - "Processing 1746 of 1000 on page 2\n", - "Processing 1747 of 1000 on page 2\n", - "Processing 1748 of 1000 on page 2\n", - "Processing 1749 of 1000 on page 2\n", - "Processing 1750 of 1000 on page 2\n", - "Processing 1751 of 1000 on page 2\n", - "Processing 1752 of 1000 on page 2\n", - "Processing 1753 of 1000 on page 2\n", - "Processing 1754 of 1000 on page 2\n", - "Processing 1755 of 1000 on page 2\n", - "Processing 1756 of 1000 on page 2\n", - "Processing 1757 of 1000 on page 2\n", - "Processing 1758 of 1000 on page 2\n", - "Processing 1759 of 1000 on page 2\n", - "Processing 1760 of 1000 on page 2\n", - "Processing 1761 of 1000 on page 2\n", - "Processing 1762 of 1000 on page 2\n", - "Processing 1763 of 1000 on page 2\n", - "Processing 1764 of 1000 on page 2\n", - "Processing 1765 of 1000 on page 2\n", - "Processing 1766 of 1000 on page 2\n", - "Processing 1767 of 1000 on page 2\n", - "Processing 1768 of 1000 on page 2\n", - "Processing 1769 of 1000 on page 2\n", - "Processing 1770 of 1000 on page 2\n", - "Processing 1771 of 1000 on page 2\n", - "Processing 1772 of 1000 on page 2\n", - "Processing 1773 of 1000 on page 2\n", - "Processing 1774 of 1000 on page 2\n", - "Processing 1775 of 1000 on page 2\n", - "Processing 1776 of 1000 on page 2\n", - "Processing 1777 of 1000 on page 2\n", - "Processing 1778 of 1000 on page 2\n", - "Processing 1779 of 1000 on page 2\n", - "Processing 1780 of 1000 on page 2\n", - "Processing 1781 of 1000 on page 2\n", - "Processing 1782 of 1000 on page 2\n", - "Processing 1783 of 1000 on page 2\n", - "Processing 1784 of 1000 on page 2\n", - "Processing 1785 of 1000 on page 2\n", - "Processing 1786 of 1000 on page 2\n", - "Processing 1787 of 1000 on page 2\n", - "Processing 1788 of 1000 on page 2\n", - "Processing 1789 of 1000 on page 2\n", - "Processing 1790 of 1000 on page 2\n", - "Processing 1791 of 1000 on page 2\n", - "Processing 1792 of 1000 on page 2\n", - "Processing 1793 of 1000 on page 2\n", - "Processing 1794 of 1000 on page 2\n", - "Processing 1795 of 1000 on page 2\n", - "Processing 1796 of 1000 on page 2\n", - "Processing 1797 of 1000 on page 2\n", - "Processing 1798 of 1000 on page 2\n", - "Processing 1799 of 1000 on page 2\n", - "Processing 1800 of 1000 on page 2\n", - "Processing 1801 of 1000 on page 2\n", - "Processing 1802 of 1000 on page 2\n", - "Processing 1803 of 1000 on page 2\n", - "Processing 1804 of 1000 on page 2\n", - "Processing 1805 of 1000 on page 2\n", - "Processing 1806 of 1000 on page 2\n", - "Processing 1807 of 1000 on page 2\n", - "Processing 1808 of 1000 on page 2\n", - "Processing 1809 of 1000 on page 2\n", - "Processing 1810 of 1000 on page 2\n", - "Processing 1811 of 1000 on page 2\n", - "Processing 1812 of 1000 on page 2\n", - "Processing 1813 of 1000 on page 2\n", - "Processing 1814 of 1000 on page 2\n", - "Processing 1815 of 1000 on page 2\n", - "Processing 1816 of 1000 on page 2\n", - "Processing 1817 of 1000 on page 2\n", - "Processing 1818 of 1000 on page 2\n", - "Processing 1819 of 1000 on page 2\n", - "Processing 1820 of 1000 on page 2\n", - "Processing 1821 of 1000 on page 2\n", - "Processing 1822 of 1000 on page 2\n", - "Processing 1823 of 1000 on page 2\n", - "Processing 1824 of 1000 on page 2\n", - "Processing 1825 of 1000 on page 2\n", - "Processing 1826 of 1000 on page 2\n", - "Processing 1827 of 1000 on page 2\n", - "Processing 1828 of 1000 on page 2\n", - "Processing 1829 of 1000 on page 2\n", - "Processing 1830 of 1000 on page 2\n", - "Processing 1831 of 1000 on page 2\n", - "Processing 1832 of 1000 on page 2\n", - "Processing 1833 of 1000 on page 2\n", - "Processing 1834 of 1000 on page 2\n", - "Processing 1835 of 1000 on page 2\n", - "Processing 1836 of 1000 on page 2\n", - "Processing 1837 of 1000 on page 2\n", - "Processing 1838 of 1000 on page 2\n", - "Processing 1839 of 1000 on page 2\n", - "Processing 1840 of 1000 on page 2\n", - "Processing 1841 of 1000 on page 2\n", - "Processing 1842 of 1000 on page 2\n", - "Processing 1843 of 1000 on page 2\n", - "Processing 1844 of 1000 on page 2\n", - "Processing 1845 of 1000 on page 2\n", - "Processing 1846 of 1000 on page 2\n", - "Processing 1847 of 1000 on page 2\n", - "Processing 1848 of 1000 on page 2\n", - "Processing 1849 of 1000 on page 2\n", - "Processing 1850 of 1000 on page 2\n", - "Processing 1851 of 1000 on page 2\n", - "Processing 1852 of 1000 on page 2\n", - "Processing 1853 of 1000 on page 2\n", - "Processing 1854 of 1000 on page 2\n", - "Processing 1855 of 1000 on page 2\n", - "Processing 1856 of 1000 on page 2\n", - "Processing 1857 of 1000 on page 2\n", - "Processing 1858 of 1000 on page 2\n", - "Processing 1859 of 1000 on page 2\n", - "Processing 1860 of 1000 on page 2\n", - "Processing 1861 of 1000 on page 2\n", - "Processing 1862 of 1000 on page 2\n", - "Processing 1863 of 1000 on page 2\n", - "Processing 1864 of 1000 on page 2\n", - "Processing 1865 of 1000 on page 2\n", - "Processing 1866 of 1000 on page 2\n", - "Processing 1867 of 1000 on page 2\n", - "Processing 1868 of 1000 on page 2\n", - "Processing 1869 of 1000 on page 2\n", - "Processing 1870 of 1000 on page 2\n", - "Processing 1871 of 1000 on page 2\n", - "Processing 1872 of 1000 on page 2\n", - "Processing 1873 of 1000 on page 2\n", - "Processing 1874 of 1000 on page 2\n", - "Processing 1875 of 1000 on page 2\n", - "Processing 1876 of 1000 on page 2\n", - "Processing 1877 of 1000 on page 2\n", - "Processing 1878 of 1000 on page 2\n", - "Processing 1879 of 1000 on page 2\n", - "Processing 1880 of 1000 on page 2\n", - "Processing 1881 of 1000 on page 2\n", - "Processing 1882 of 1000 on page 2\n", - "Processing 1883 of 1000 on page 2\n", - "Processing 1884 of 1000 on page 2\n", - "Processing 1885 of 1000 on page 2\n", - "Processing 1886 of 1000 on page 2\n", - "Processing 1887 of 1000 on page 2\n", - "Processing 1888 of 1000 on page 2\n", - "Processing 1889 of 1000 on page 2\n", - "Processing 1890 of 1000 on page 2\n", - "Processing 1891 of 1000 on page 2\n", - "Processing 1892 of 1000 on page 2\n", - "Processing 1893 of 1000 on page 2\n", - "Processing 1894 of 1000 on page 2\n", - "Processing 1895 of 1000 on page 2\n", - "Processing 1896 of 1000 on page 2\n", - "Processing 1897 of 1000 on page 2\n", - "Processing 1898 of 1000 on page 2\n", - "Processing 1899 of 1000 on page 2\n", - "Processing 1900 of 1000 on page 2\n", - "Processing 1901 of 1000 on page 2\n", - "Processing 1902 of 1000 on page 2\n", - "Processing 1903 of 1000 on page 2\n", - "Processing 1904 of 1000 on page 2\n", - "Processing 1905 of 1000 on page 2\n", - "Processing 1906 of 1000 on page 2\n", - "Processing 1907 of 1000 on page 2\n", - "Processing 1908 of 1000 on page 2\n", - "Processing 1909 of 1000 on page 2\n", - "Processing 1910 of 1000 on page 2\n", - "Processing 1911 of 1000 on page 2\n", - "Processing 1912 of 1000 on page 2\n", - "Processing 1913 of 1000 on page 2\n", - "Processing 1914 of 1000 on page 2\n", - "Processing 1915 of 1000 on page 2\n", - "Processing 1916 of 1000 on page 2\n", - "Processing 1917 of 1000 on page 2\n", - "Processing 1918 of 1000 on page 2\n", - "Processing 1919 of 1000 on page 2\n", - "Processing 1920 of 1000 on page 2\n", - "Processing 1921 of 1000 on page 2\n", - "Processing 1922 of 1000 on page 2\n", - "Processing 1923 of 1000 on page 2\n", - "Processing 1924 of 1000 on page 2\n", - "Processing 1925 of 1000 on page 2\n", - "Processing 1926 of 1000 on page 2\n", - "Processing 1927 of 1000 on page 2\n", - "Processing 1928 of 1000 on page 2\n", - "Processing 1929 of 1000 on page 2\n", - "Processing 1930 of 1000 on page 2\n", - "Processing 1931 of 1000 on page 2\n", - "Processing 1932 of 1000 on page 2\n", - "Processing 1933 of 1000 on page 2\n", - "Processing 1934 of 1000 on page 2\n", - "Processing 1935 of 1000 on page 2\n", - "Processing 1936 of 1000 on page 2\n", - "Processing 1937 of 1000 on page 2\n", - "Processing 1938 of 1000 on page 2\n", - "Processing 1939 of 1000 on page 2\n", - "Processing 1940 of 1000 on page 2\n", - "Processing 1941 of 1000 on page 2\n", - "Processing 1942 of 1000 on page 2\n", - "Processing 1943 of 1000 on page 2\n", - "Processing 1944 of 1000 on page 2\n", - "Processing 1945 of 1000 on page 2\n", - "Processing 1946 of 1000 on page 2\n", - "Processing 1947 of 1000 on page 2\n", - "Processing 1948 of 1000 on page 2\n", - "Processing 1949 of 1000 on page 2\n", - "Processing 1950 of 1000 on page 2\n", - "Processing 1951 of 1000 on page 2\n", - "Processing 1952 of 1000 on page 2\n", - "Processing 1953 of 1000 on page 2\n", - "Processing 1954 of 1000 on page 2\n", - "Processing 1955 of 1000 on page 2\n", - "Processing 1956 of 1000 on page 2\n", - "Processing 1957 of 1000 on page 2\n", - "Processing 1958 of 1000 on page 2\n", - "Processing 1959 of 1000 on page 2\n", - "Processing 1960 of 1000 on page 2\n", - "Processing 1961 of 1000 on page 2\n", - "Processing 1962 of 1000 on page 2\n", - "Processing 1963 of 1000 on page 2\n", - "Processing 1964 of 1000 on page 2\n", - "Processing 1965 of 1000 on page 2\n", - "Processing 1966 of 1000 on page 2\n", - "Processing 1967 of 1000 on page 2\n", - "Processing 1968 of 1000 on page 2\n", - "Processing 1969 of 1000 on page 2\n", - "Processing 1970 of 1000 on page 2\n", - "Processing 1971 of 1000 on page 2\n", - "Processing 1972 of 1000 on page 2\n", - "Processing 1973 of 1000 on page 2\n", - "Processing 1974 of 1000 on page 2\n", - "Processing 1975 of 1000 on page 2\n", - "Processing 1976 of 1000 on page 2\n", - "Processing 1977 of 1000 on page 2\n", - "Processing 1978 of 1000 on page 2\n", - "Processing 1979 of 1000 on page 2\n", - "Processing 1980 of 1000 on page 2\n", - "Processing 1981 of 1000 on page 2\n", - "Processing 1982 of 1000 on page 2\n", - "Processing 1983 of 1000 on page 2\n", - "Processing 1984 of 1000 on page 2\n", - "Processing 1985 of 1000 on page 2\n", - "Processing 1986 of 1000 on page 2\n", - "Processing 1987 of 1000 on page 2\n", - "Processing 1988 of 1000 on page 2\n", - "Processing 1989 of 1000 on page 2\n", - "Processing 1990 of 1000 on page 2\n", - "Processing 1991 of 1000 on page 2\n", - "Processing 1992 of 1000 on page 2\n", - "Processing 1993 of 1000 on page 2\n", - "Processing 1994 of 1000 on page 2\n", - "Processing 1995 of 1000 on page 2\n", - "Processing 1996 of 1000 on page 2\n", - "Processing 1997 of 1000 on page 2\n", - "Processing 1998 of 1000 on page 2\n", - "Processing 1999 of 1000 on page 2\n", - "Processing 2000 of 1000 on page 2\n", - "Processing 2001 of 153 on page 3\n", - "Processing 2002 of 153 on page 3\n", - "Processing 2003 of 153 on page 3\n", - "Processing 2004 of 153 on page 3\n", - "Processing 2005 of 153 on page 3\n", - "Processing 2006 of 153 on page 3\n", - "Processing 2007 of 153 on page 3\n", - "Processing 2008 of 153 on page 3\n", - "Processing 2009 of 153 on page 3\n", - "Processing 2010 of 153 on page 3\n", - "Processing 2011 of 153 on page 3\n", - "Processing 2012 of 153 on page 3\n", - "Processing 2013 of 153 on page 3\n", - "Processing 2014 of 153 on page 3\n", - "Processing 2015 of 153 on page 3\n", - "Processing 2016 of 153 on page 3\n", - "Processing 2017 of 153 on page 3\n", - "Processing 2018 of 153 on page 3\n", - "Processing 2019 of 153 on page 3\n", - "Processing 2020 of 153 on page 3\n", - "Processing 2021 of 153 on page 3\n", - "Processing 2022 of 153 on page 3\n", - "Processing 2023 of 153 on page 3\n", - "Processing 2024 of 153 on page 3\n", - "Processing 2025 of 153 on page 3\n", - "Processing 2026 of 153 on page 3\n", - "Processing 2027 of 153 on page 3\n", - "Processing 2028 of 153 on page 3\n", - "Processing 2029 of 153 on page 3\n", - "Processing 2030 of 153 on page 3\n", - "Processing 2031 of 153 on page 3\n", - "Processing 2032 of 153 on page 3\n", - "Processing 2033 of 153 on page 3\n", - "Processing 2034 of 153 on page 3\n", - "Processing 2035 of 153 on page 3\n", - "Processing 2036 of 153 on page 3\n", - "Processing 2037 of 153 on page 3\n", - "Processing 2038 of 153 on page 3\n", - "Processing 2039 of 153 on page 3\n", - "Processing 2040 of 153 on page 3\n", - "Processing 2041 of 153 on page 3\n", - "Processing 2042 of 153 on page 3\n", - "Processing 2043 of 153 on page 3\n", - "Processing 2044 of 153 on page 3\n", - "Processing 2045 of 153 on page 3\n", - "Processing 2046 of 153 on page 3\n", - "Processing 2047 of 153 on page 3\n", - "Processing 2048 of 153 on page 3\n", - "Processing 2049 of 153 on page 3\n", - "Processing 2050 of 153 on page 3\n", - "Processing 2051 of 153 on page 3\n", - "Processing 2052 of 153 on page 3\n", - "Processing 2053 of 153 on page 3\n", - "Processing 2054 of 153 on page 3\n", - "Processing 2055 of 153 on page 3\n", - "Processing 2056 of 153 on page 3\n", - "Processing 2057 of 153 on page 3\n", - "Processing 2058 of 153 on page 3\n", - "Processing 2059 of 153 on page 3\n", - "Processing 2060 of 153 on page 3\n", - "Processing 2061 of 153 on page 3\n", - "Processing 2062 of 153 on page 3\n", - "Processing 2063 of 153 on page 3\n", - "Processing 2064 of 153 on page 3\n", - "Processing 2065 of 153 on page 3\n", - "Processing 2066 of 153 on page 3\n", - "Processing 2067 of 153 on page 3\n", - "Processing 2068 of 153 on page 3\n", - "Processing 2069 of 153 on page 3\n", - "Processing 2070 of 153 on page 3\n", - "Processing 2071 of 153 on page 3\n", - "Processing 2072 of 153 on page 3\n", - "Processing 2073 of 153 on page 3\n", - "Processing 2074 of 153 on page 3\n", - "Processing 2075 of 153 on page 3\n", - "Processing 2076 of 153 on page 3\n", - "Processing 2077 of 153 on page 3\n", - "Processing 2078 of 153 on page 3\n", - "Processing 2079 of 153 on page 3\n", - "Processing 2080 of 153 on page 3\n", - "Processing 2081 of 153 on page 3\n", - "Processing 2082 of 153 on page 3\n", - "Processing 2083 of 153 on page 3\n", - "Processing 2084 of 153 on page 3\n", - "Processing 2085 of 153 on page 3\n", - "Processing 2086 of 153 on page 3\n", - "Processing 2087 of 153 on page 3\n", - "Processing 2088 of 153 on page 3\n", - "Processing 2089 of 153 on page 3\n", - "Processing 2090 of 153 on page 3\n", - "Processing 2091 of 153 on page 3\n", - "Processing 2092 of 153 on page 3\n", - "Processing 2093 of 153 on page 3\n", - "Processing 2094 of 153 on page 3\n", - "Processing 2095 of 153 on page 3\n", - "Processing 2096 of 153 on page 3\n", - "Processing 2097 of 153 on page 3\n", - "Processing 2098 of 153 on page 3\n", - "Processing 2099 of 153 on page 3\n", - "Processing 2100 of 153 on page 3\n", - "Processing 2101 of 153 on page 3\n", - "Processing 2102 of 153 on page 3\n", - "Processing 2103 of 153 on page 3\n", - "Processing 2104 of 153 on page 3\n", - "Processing 2105 of 153 on page 3\n", - "Processing 2106 of 153 on page 3\n", - "Processing 2107 of 153 on page 3\n", - "Processing 2108 of 153 on page 3\n", - "Processing 2109 of 153 on page 3\n", - "Processing 2110 of 153 on page 3\n", - "Processing 2111 of 153 on page 3\n", - "Processing 2112 of 153 on page 3\n", - "Processing 2113 of 153 on page 3\n", - "Processing 2114 of 153 on page 3\n", - "Processing 2115 of 153 on page 3\n", - "Processing 2116 of 153 on page 3\n", - "Processing 2117 of 153 on page 3\n", - "Processing 2118 of 153 on page 3\n", - "Processing 2119 of 153 on page 3\n", - "Processing 2120 of 153 on page 3\n", - "Processing 2121 of 153 on page 3\n", - "Processing 2122 of 153 on page 3\n", - "Processing 2123 of 153 on page 3\n", - "Processing 2124 of 153 on page 3\n", - "Processing 2125 of 153 on page 3\n", - "Processing 2126 of 153 on page 3\n", - "Processing 2127 of 153 on page 3\n", - "Processing 2128 of 153 on page 3\n", - "Processing 2129 of 153 on page 3\n", - "Processing 2130 of 153 on page 3\n", - "Processing 2131 of 153 on page 3\n", - "Processing 2132 of 153 on page 3\n", - "Processing 2133 of 153 on page 3\n", - "Processing 2134 of 153 on page 3\n", - "Processing 2135 of 153 on page 3\n", - "Processing 2136 of 153 on page 3\n", - "Processing 2137 of 153 on page 3\n", - "Processing 2138 of 153 on page 3\n", - "Processing 2139 of 153 on page 3\n", - "Processing 2140 of 153 on page 3\n", - "Processing 2141 of 153 on page 3\n", - "Processing 2142 of 153 on page 3\n", - "Processing 2143 of 153 on page 3\n", - "Processing 2144 of 153 on page 3\n", - "Processing 2145 of 153 on page 3\n", - "Processing 2146 of 153 on page 3\n", - "Processing 2147 of 153 on page 3\n", - "Processing 2148 of 153 on page 3\n", - "Processing 2149 of 153 on page 3\n", - "Processing 2150 of 153 on page 3\n", - "Processing 2151 of 153 on page 3\n", - "Processing 2152 of 153 on page 3\n", - "Processing 2153 of 153 on page 3\n", - "usgs_elev_tables load completed\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Has appx 2,150 HUCs to process, but this section goes quickly.\n", @@ -3347,40 +1092,43 @@ " 'Delimiter': '/'}\n", "page_iterator = paginator.paginate(**operation_parameters)\n", "page_count = 0\n", - "for page in page_iterator:\n", + "num_pages = len(page_iterator)\n", + "print(f\"num of pages is {num_pages}\")\n", + "# for page in page_iterator:\n", " \n", - " prefix_objects = page['CommonPrefixes']\n", - " for i, prefix_obj in enumerate(prefix_objects):\n", - " print(f\"Processing {(i+1) + (1000 * page_count)} of\"\n", - " f\"{len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\")\n", - " huc_prefix = prefix_obj.get(\"Prefix\")\n", - " usgs_elev_table_key = f'{huc_prefix}usgs_elev_table.csv'\n", - " try:\n", - " uet = S3_CLIENT.get_object(\n", - " Bucket=FIM_BUCKET, \n", - " Key=usgs_elev_table_key\n", - " )['Body']\n", - " uet_df = pd.read_csv(uet, header=0, usecols=uet_usecols)\n", - " uet_df['fim_version'] = PUBLIC_FIM_VERSION\n", - " uet_df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", - " uet_df.to_sql(\n", - " con=VIZ_DB_ENGINE,\n", - " dtype={\n", - " \"location_id\": Text(),\n", - " \"nws_data_huc\": Text()\n", - " },\n", - " schema='derived',\n", - " name='usgs_elev_table',\n", - " index=False, \n", - " if_exists='append'\n", - " )\n", - " except Exception as e:\n", - " if \"NoSuchKey\" in str(e):\n", - " pass\n", - " else:\n", - " raise e\n", - "\n", - " page_count += 1\n", + "# prefix_objects = page['CommonPrefixes']\n", + "# for i, prefix_obj in enumerate(prefix_objects):\n", + "# display_index = (i + 1) + (1000 * page_count)\n", + "# print(f\"Processing {i+1} of\"\n", + "# f\" {len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\")\n", + "# huc_prefix = prefix_obj.get(\"Prefix\")\n", + "# usgs_elev_table_key = f'{huc_prefix}usgs_elev_table.csv'\n", + "# try:\n", + "# uet = S3_CLIENT.get_object(\n", + "# Bucket=FIM_BUCKET, \n", + "# Key=usgs_elev_table_key\n", + "# )['Body']\n", + "# uet_df = pd.read_csv(uet, header=0, usecols=uet_usecols)\n", + "# uet_df['fim_version'] = PUBLIC_FIM_VERSION\n", + "# uet_df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", + "# uet_df.to_sql(\n", + "# con=VIZ_DB_ENGINE,\n", + "# dtype={\n", + "# \"location_id\": Text(),\n", + "# \"nws_data_huc\": Text()\n", + "# },\n", + "# schema='derived',\n", + "# name='usgs_elev_table',\n", + "# index=False, \n", + "# if_exists='append'\n", + "# )\n", + "# except Exception as e:\n", + "# if \"NoSuchKey\" in str(e):\n", + "# pass\n", + "# else:\n", + "# raise e\n", + "\n", + "# page_count += 1\n", " \n", " \n", "print(\"usgs_elev_tables load completed\")\n" @@ -3396,51 +1144,12 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "33026eb6-fbc7-47f7-a44b-f9cea8f0911b", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hydrotable reloaded - started\n", - "Processing 1 of 0 on page 1 (1000 per page)\n", - "Processing 1002 of 1000 on page 2 (1000 per page)\n", - "Processing 2003 of 2000 on page 3 (1000 per page)\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[16], line 46\u001b[0m\n\u001b[1;32m 41\u001b[0m ht \u001b[38;5;241m=\u001b[39m S3_CLIENT\u001b[38;5;241m.\u001b[39mget_object(\n\u001b[1;32m 42\u001b[0m Bucket\u001b[38;5;241m=\u001b[39mFIM_BUCKET,\n\u001b[1;32m 43\u001b[0m Key\u001b[38;5;241m=\u001b[39mhydro_table_key\n\u001b[1;32m 44\u001b[0m )[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mBody\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# print(\"...Reading with pandas...\")\u001b[39;00m\n\u001b[0;32m---> 46\u001b[0m ht_df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mht\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheader\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43musecols\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mht_usecols\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;66;03m# print('...Writing to db...')\u001b[39;00m\n\u001b[1;32m 48\u001b[0m ht_df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfim_version\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m PUBLIC_FIM_VERSION\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1026\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m 1013\u001b[0m kwds_defaults \u001b[38;5;241m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m 1014\u001b[0m dialect,\n\u001b[1;32m 1015\u001b[0m delimiter,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1022\u001b[0m dtype_backend\u001b[38;5;241m=\u001b[39mdtype_backend,\n\u001b[1;32m 1023\u001b[0m )\n\u001b[1;32m 1024\u001b[0m kwds\u001b[38;5;241m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m-> 1026\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:626\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 623\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parser\n\u001b[1;32m 625\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m parser:\n\u001b[0;32m--> 626\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mparser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1923\u001b[0m, in \u001b[0;36mTextFileReader.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 1916\u001b[0m nrows \u001b[38;5;241m=\u001b[39m validate_integer(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnrows\u001b[39m\u001b[38;5;124m\"\u001b[39m, nrows)\n\u001b[1;32m 1917\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1918\u001b[0m \u001b[38;5;66;03m# error: \"ParserBase\" has no attribute \"read\"\u001b[39;00m\n\u001b[1;32m 1919\u001b[0m (\n\u001b[1;32m 1920\u001b[0m index,\n\u001b[1;32m 1921\u001b[0m columns,\n\u001b[1;32m 1922\u001b[0m col_dict,\n\u001b[0;32m-> 1923\u001b[0m ) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# type: ignore[attr-defined]\u001b[39;49;00m\n\u001b[1;32m 1924\u001b[0m \u001b[43m \u001b[49m\u001b[43mnrows\u001b[49m\n\u001b[1;32m 1925\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1926\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 1927\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/io/parsers/c_parser_wrapper.py:234\u001b[0m, in \u001b[0;36mCParserWrapper.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 233\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlow_memory:\n\u001b[0;32m--> 234\u001b[0m chunks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_low_memory\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;66;03m# destructive to chunks\u001b[39;00m\n\u001b[1;32m 236\u001b[0m data \u001b[38;5;241m=\u001b[39m _concatenate_chunks(chunks)\n", - "File \u001b[0;32mparsers.pyx:838\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader.read_low_memory\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32mparsers.pyx:905\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32mparsers.pyx:874\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._tokenize_rows\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32mparsers.pyx:891\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._check_tokenize_status\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32mparsers.pyx:2053\u001b[0m, in \u001b[0;36mpandas._libs.parsers.raise_parser_error\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/response.py:99\u001b[0m, in \u001b[0;36mStreamingBody.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Read at most amt bytes from the stream.\u001b[39;00m\n\u001b[1;32m 95\u001b[0m \n\u001b[1;32m 96\u001b[0m \u001b[38;5;124;03mIf the amt argument is omitted, read all data.\u001b[39;00m\n\u001b[1;32m 97\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 99\u001b[0m chunk \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raw_stream\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 100\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m URLLib3ReadTimeoutError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 101\u001b[0m \u001b[38;5;66;03m# TODO: the url will be None as urllib3 isn't setting it yet\u001b[39;00m\n\u001b[1;32m 102\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ReadTimeoutError(endpoint_url\u001b[38;5;241m=\u001b[39me\u001b[38;5;241m.\u001b[39murl, error\u001b[38;5;241m=\u001b[39me)\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/urllib3/response.py:949\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt, decode_content, cache_content)\u001b[0m\n\u001b[1;32m 946\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m amt:\n\u001b[1;32m 947\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer\u001b[38;5;241m.\u001b[39mget(amt)\n\u001b[0;32m--> 949\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raw_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 951\u001b[0m flush_decoder \u001b[38;5;241m=\u001b[39m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (amt \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data)\n\u001b[1;32m 953\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/urllib3/response.py:873\u001b[0m, in \u001b[0;36mHTTPResponse._raw_read\u001b[0;34m(self, amt, read1)\u001b[0m\n\u001b[1;32m 870\u001b[0m fp_closed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclosed\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 872\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_error_catcher():\n\u001b[0;32m--> 873\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mread1\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mread1\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m fp_closed \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 874\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[1;32m 875\u001b[0m \u001b[38;5;66;03m# Platform-specific: Buggy versions of Python.\u001b[39;00m\n\u001b[1;32m 876\u001b[0m \u001b[38;5;66;03m# Close the connection when no data is returned\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 881\u001b[0m \u001b[38;5;66;03m# not properly close the connection in all cases. There is\u001b[39;00m\n\u001b[1;32m 882\u001b[0m \u001b[38;5;66;03m# no harm in redundantly calling close.\u001b[39;00m\n\u001b[1;32m 883\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mclose()\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/urllib3/response.py:856\u001b[0m, in \u001b[0;36mHTTPResponse._fp_read\u001b[0;34m(self, amt, read1)\u001b[0m\n\u001b[1;32m 853\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread1(amt) \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread1()\n\u001b[1;32m 854\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 855\u001b[0m \u001b[38;5;66;03m# StringIO doesn't like amt=None\u001b[39;00m\n\u001b[0;32m--> 856\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread()\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/http/client.py:466\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength:\n\u001b[1;32m 464\u001b[0m \u001b[38;5;66;03m# clip the read to the \"end of response\"\u001b[39;00m\n\u001b[1;32m 465\u001b[0m amt \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength\n\u001b[0;32m--> 466\u001b[0m s \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 467\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m s \u001b[38;5;129;01mand\u001b[39;00m amt:\n\u001b[1;32m 468\u001b[0m \u001b[38;5;66;03m# Ideally, we would raise IncompleteRead if the content-length\u001b[39;00m\n\u001b[1;32m 469\u001b[0m \u001b[38;5;66;03m# wasn't satisfied, but it might break compatibility.\u001b[39;00m\n\u001b[1;32m 470\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_conn()\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/socket.py:705\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m 704\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 705\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 706\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m 707\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/ssl.py:1307\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m 1303\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 1304\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 1305\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m 1306\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1307\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1308\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1309\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n", - "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/ssl.py:1163\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m 1161\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1162\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1163\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1164\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "\n", "# Takes appx 5.75 to 6 hrs to run\n", From 351e0a40cd4e0ddabe2c8c4af0e1ab187dc2ba2d Mon Sep 17 00:00:00 2001 From: RobH Date: Tue, 19 Nov 2024 02:59:15 +0000 Subject: [PATCH 11/18] after branch 0 catchments --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 2785 +++++++++++++++-- 1 file changed, 2519 insertions(+), 266 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 87a16d2b..3f5835fc 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -165,7 +165,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 29, "id": "e944eff3-6023-48f4-9f57-27304a240447", "metadata": { "tags": [] @@ -449,6 +449,11 @@ "execution_count": 7, "id": "6a698067", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true, + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -518,6 +523,11 @@ "execution_count": 8, "id": "f6d6ee69", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true, + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -554,6 +564,11 @@ "execution_count": 9, "id": "a4f89d9a", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true, + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -589,6 +604,11 @@ "execution_count": 10, "id": "791d1a8b", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true, + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -624,6 +644,11 @@ "execution_count": 11, "id": "1bb87128", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true, + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -659,6 +684,11 @@ "execution_count": 12, "id": "4832e4e0", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true, + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -694,6 +724,11 @@ "execution_count": 13, "id": "187e83bc-ebbe-4615-a046-e0ef7b09ad3b", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true, + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -749,20 +784,140 @@ ] }, { - "cell_type": "markdown", - "id": "b4574e4b-1569-4362-8d3f-42f4319752a1", + "cell_type": "code", + "execution_count": 38, + "id": "a7495759", "metadata": { "tags": [] }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "get_catchment_pipepline_input loaded\n" + ] + } + ], + "source": [ + "\n", + "def get_catchment_pipepline_input(branch_key):\n", + "\n", + " catchment_name_key = f\"catchments_{branch_key}_branches\"\n", + "\n", + " pipeline_input = {\n", + " \"configuration\": \"reference\",\n", + " \"job_type\": \"auto\",\n", + " \"data_type\": \"channel\",\n", + " \"keep_raw\": False,\n", + " \"reference_time\": datetime.now().strftime('%Y-%m-%d 00:00:00'),\n", + " \"configuration_data_flow\": {\n", + " \"db_max_flows\": [],\n", + " \"db_ingest_groups\": [],\n", + " \"python_preprocessing\": []\n", + " },\n", + " \"pipeline_products\": [\n", + " {\n", + " \"product\": f\"static_hand_{catchment_name_key}\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": f\"{catchment_name_key}\",\n", + " \"target_table\": f\"fim_catchments.{catchment_name_key}\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": f\"{catchment_name_key}\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " f\"static_hand_{catchment_name_key}_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " },\n", + " {\n", + " \"product\": f\"static_hand_{catchment_name_key}_hi\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": f\"{catchment_name_key}_hi\",\n", + " \"target_table\": f\"fim_catchments.{catchment_name_key}_hi\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": f\"{catchment_name_key}_hi\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " f\"static_hand_{catchment_name_key}_hi_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " },\n", + " {\n", + " \"product\": f\"static_hand_{catchment_name_key}_prvi\",\n", + " \"configuration\": \"reference\",\n", + " \"product_type\": \"fim\",\n", + " \"run\": True,\n", + " \"fim_configs\": [\n", + " {\n", + " \"name\": f\"{catchment_name_key}_prvi\",\n", + " \"target_table\": f\"fim_catchments.{catchment_name_key}_prvi\",\n", + " \"fim_type\": \"hand\",\n", + " \"sql_file\": f\"{catchment_name_key}_prvi\"\n", + " }\n", + " ],\n", + " \"services\": [\n", + " f\"static_hand_{catchment_name_key}_prvi_noaa\"\n", + " ],\n", + " \"raster_outputs\": {\n", + " \"output_bucket\": \"\",\n", + " \"output_raster_workspaces\": []\n", + " },\n", + " \"postprocess_sql\": [],\n", + " \"product_summaries\": [],\n", + " \"python_preprocesing_dependent\": False\n", + " }\n", + " ],\n", + " \"sql_rename_dict\": {},\n", + " \"logging_info\": {\n", + " \"Timestamp\": int(datetime.now().timestamp())\n", + " }\n", + " }\n", + "\n", + " return pipeline_input\n", + "\n", + "print(\"get_catchment_pipepline_input loaded\")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "f1fa1947-39af-49b4-8970-579ec4001d1c", + "metadata": {}, "source": [ "### 6a - Branch 0 Catchments. Wait until it is done before kicking off the next GMS (Level Path) catchments load a bit lower. ###" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 39, "id": "c77d6e5b-f5fb-422f-a6c3-0f53743cd631", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -776,17 +931,15 @@ } ], "source": [ - "# TODO: Add backups to these (4.4.0.0) (already not available for 4.4.0.0)\n", "\n", - "\n", - "# TODO: We likely need to keep the schema, so trun is fine for now, but eventually, get a lsit of the indexes and re-build \n", - "# indexes each time as/if needed. Granted these tables are loaded via Lambdas, so I am not sure how indexes will play into that\n", + "# Npte: The three db's here were renamed from:\n", + "# \"branch_0_catchments\", \"branch_0_catchments_hi\", \"branch_0_catchments_prvi\",\n", "\n", "sf.execute_sql('''\n", "TRUNCATE \n", - " fim_catchments.branch_0_catchments, \n", - " fim_catchments.branch_0_catchments_hi, \n", - " fim_catchments.branch_0_catchments_prvi;\n", + " fim_catchments.catchments_0_branches,\n", + " fim_catchments.catchments_0_branches_hi,\n", + " fim_catchments.catchments_0_branches_prvi;\n", "''', db_type=\"egis\")\n", "\n", "print(\"Catchment Truncation for Branch 0 Done\")\n", @@ -795,8 +948,8 @@ }, { "cell_type": "code", - "execution_count": 15, - "id": "a7495759", + "execution_count": 40, + "id": "8c2806b8-5b9f-435c-b0ae-a28f9224b938", "metadata": { "tags": [] }, @@ -805,114 +958,25 @@ "name": "stdout", "output_type": "stream", "text": [ - "Catchments Branch 0 load kicked off. Last runtime: 23:38.019. Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - sagemaker_0_catchments_20241116T1945\n" + "Catchments Branch 0 load kicked off. Last runtime: 23:38.019. Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - hv_ti_data_loads_catchments_branch_0_20241118T2321\n" ] } ], "source": [ - "pipeline_input = {\n", - " \"configuration\": \"reference\",\n", - " \"job_type\": \"auto\",\n", - " \"data_type\": \"channel\",\n", - " \"keep_raw\": False,\n", - " \"reference_time\": datetime.now().strftime('%Y-%m-%d 00:00:00'),\n", - " \"configuration_data_flow\": {\n", - " \"db_max_flows\": [],\n", - " \"db_ingest_groups\": [],\n", - " \"python_preprocessing\": []\n", - " },\n", - " \"pipeline_products\": [\n", - " {\n", - " \"product\": \"static_hand_catchments_0_branches\",\n", - " \"configuration\": \"reference\",\n", - " \"product_type\": \"fim\",\n", - " \"run\": True,\n", - " \"fim_configs\": [\n", - " {\n", - " \"name\": \"catchments_0_branches\",\n", - " \"target_table\": \"fim_catchments.branch_0_catchments\",\n", - " \"fim_type\": \"hand\",\n", - " \"sql_file\": \"catchments_0_branches\"\n", - " }\n", - " ],\n", - " \"services\": [\n", - " \"static_hand_catchments_0_branches_noaa\"\n", - " ],\n", - " \"raster_outputs\": {\n", - " \"output_bucket\": \"\",\n", - " \"output_raster_workspaces\": []\n", - " },\n", - " \"postprocess_sql\": [],\n", - " \"product_summaries\": [],\n", - " \"python_preprocesing_dependent\": False\n", - " },\n", - " {\n", - " \"product\": \"static_hand_catchments_0_branches_hi\",\n", - " \"configuration\": \"reference\",\n", - " \"product_type\": \"fim\",\n", - " \"run\": True,\n", - " \"fim_configs\": [\n", - " {\n", - " \"name\": \"catchments_0_branches_hi\",\n", - " \"target_table\": \"fim_catchments.branch_0_catchments_hi\",\n", - " \"fim_type\": \"hand\",\n", - " \"sql_file\": \"catchments_0_branches_hi\"\n", - " }\n", - " ],\n", - " \"services\": [\n", - " \"static_hand_catchments_0_branches_hi_noaa\"\n", - " ],\n", - " \"raster_outputs\": {\n", - " \"output_bucket\": \"\",\n", - " \"output_raster_workspaces\": []\n", - " },\n", - " \"postprocess_sql\": [],\n", - " \"product_summaries\": [],\n", - " \"python_preprocesing_dependent\": False\n", - " },\n", - " {\n", - " \"product\": \"static_hand_catchments_0_branches_prvi\",\n", - " \"configuration\": \"reference\",\n", - " \"product_type\": \"fim\",\n", - " \"run\": True,\n", - " \"fim_configs\": [\n", - " {\n", - " \"name\": \"catchments_0_branches_prvi\",\n", - " \"target_table\": \"fim_catchments.branch_0_catchments_prvi\",\n", - " \"fim_type\": \"hand\",\n", - " \"sql_file\": \"catchments_0_branches_prvi\"\n", - " }\n", - " ],\n", - " \"services\": [\n", - " \"static_hand_catchments_0_branches_prvi_noaa\"\n", - " ],\n", - " \"raster_outputs\": {\n", - " \"output_bucket\": \"\",\n", - " \"output_raster_workspaces\": []\n", - " },\n", - " \"postprocess_sql\": [],\n", - " \"product_summaries\": [],\n", - " \"python_preprocesing_dependent\": False\n", - " }\n", - " ],\n", - " \"sql_rename_dict\": {},\n", - " \"logging_info\": {\n", - " \"Timestamp\": int(datetime.now().timestamp())\n", - " }\n", - "}\n", "\n", - "pipeline_name = f\"hv_ti_data_loads_sage_0_catchments_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_catchments_branch_0_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", - "# TODO: For later... fix fim_version value and add model_version column. current fim_version vlaue is showing 4.5.2.11\n", + "pipeline_input = get_catchment_pipepline_input(\"0\")\n", + "# print(pipeline_input)\n", "\n", "STEPFUNCTION_CLIENT.start_execution(\n", - " stateMachineArn = PIPELINE_ARN,\n", - " name = pipeline_name,\n", - " input= json.dumps(pipeline_input)\n", + " stateMachineArn=PIPELINE_ARN,\n", + " name=pipeline_name,\n", + " input=json.dumps(pipeline_input)\n", ")\n", "\n", - "print(\"Catchments Branch 0 load kicked off. Last runtime: 23:38.019. \"\n", - " f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - {pipeline_name}\")\n" + "print(\"Catchments Branch 0 load kicked off. Takes appx 25 mins (depending on other processess) \"\n", + " f\"Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - {pipeline_name}\")" ] }, { @@ -925,21 +989,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "id": "3ea2a5be-9936-4abe-832a-8582f3c3af64", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Catchment Truncation for GMS (Level Path) Branchs Done\n" + ] + } + ], "source": [ - "\n", - "# TODO: Add backups to these\n", "\n", "sf.execute_sql('''\n", "TRUNCATE\n", - " fim_catchments.branch_gms_catchments,\n", - " fim_catchments.branch_gms_catchments_hi,\n", - " fim_catchments.branch_gms_catchments_prvi;\n", + " fim_catchments.catchments_gms_branches,\n", + " fim_catchments.catchments_gms_branches_hi,\n", + " fim_catchments.catchments_gms_branches_prvi;\n", "''', db_type=\"egis\")\n", "\n", "print(\"Catchment Truncation for GMS (Level Path) Branchs Done\")" @@ -947,116 +1017,138 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "id": "9ae8917f", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Catchments GMS Branches (Level Paths / non branch 0) load kicked off. Takes appx 25 mins (depending on other processess)\n", + " Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - hv_ti_data_loads_catchments_gms_20241119T0255\n" + ] + } + ], "source": [ - "pipeline_input = {\n", - " \"configuration\": \"reference\",\n", - " \"job_type\": \"auto\",\n", - " \"data_type\": \"channel\",\n", - " \"keep_raw\": False,\n", - " \"reference_time\": datetime.now().strftime('%Y-%m-%d 00:00:00'),\n", - " \"configuration_data_flow\": {\n", - " \"db_max_flows\": [],\n", - " \"db_ingest_groups\": [],\n", - " \"python_preprocessing\": []\n", - " },\n", - " \"pipeline_products\": [\n", - " {\n", - " \"product\": \"static_hand_catchments_gms_branches\",\n", - " \"configuration\": \"reference\",\n", - " \"product_type\": \"fim\",\n", - " \"run\": True,\n", - " \"fim_configs\": [\n", - " {\n", - " \"name\": \"catchments_gms_branches\",\n", - " \"target_table\": \"fim_catchments.branch_gms_catchments\",\n", - " \"fim_type\": \"hand\",\n", - " \"sql_file\": \"catchments_gms_branches\"\n", - " }\n", - " ],\n", - " \"services\": [\n", - " \"static_hand_catchments_gms_branches_noaa\"\n", - " ],\n", - " \"raster_outputs\": {\n", - " \"output_bucket\": \"\",\n", - " \"output_raster_workspaces\": []\n", - " },\n", - " \"postprocess_sql\": [],\n", - " \"product_summaries\": [],\n", - " \"python_preprocesing_dependent\": False\n", - " },\n", - " {\n", - " \"product\": \"static_hand_catchments_gms_branches_hi\",\n", - " \"configuration\": \"reference\",\n", - " \"product_type\": \"fim\",\n", - " \"run\": True,\n", - " \"fim_configs\": [\n", - " {\n", - " \"name\": \"catchments_gms_branches_hi\",\n", - " \"target_table\": \"fim_catchments.branch_gms_catchments_hi\",\n", - " \"fim_type\": \"hand\",\n", - " \"sql_file\": \"catchments_gms_branches_hi\"\n", - " }\n", - " ],\n", - " \"services\": [\n", - " \"static_hand_catchments_gms_branches_hi_noaa\"\n", - " ],\n", - " \"raster_outputs\": {\n", - " \"output_bucket\": \"\",\n", - " \"output_raster_workspaces\": []\n", - " },\n", - " \"postprocess_sql\": [],\n", - " \"product_summaries\": [],\n", - " \"python_preprocesing_dependent\": False\n", - " },\n", - " {\n", - " \"product\": \"static_hand_catchments_gms_branches_prvi\",\n", - " \"configuration\": \"reference\",\n", - " \"product_type\": \"fim\",\n", - " \"run\": True,\n", - " \"fim_configs\": [\n", - " {\n", - " \"name\": \"catchments_gms_branches_prvi\",\n", - " \"target_table\": \"fim_catchments.branch_gms_catchments_prvi\",\n", - " \"fim_type\": \"hand\",\n", - " \"sql_file\": \"catchments_gms_branches_prvi\"\n", - " }\n", - " ],\n", - " \"services\": [\n", - " \"static_hand_catchments_gms_branches_prvi_noaa\"\n", - " ],\n", - " \"raster_outputs\": {\n", - " \"output_bucket\": \"\",\n", - " \"output_raster_workspaces\": []\n", - " },\n", - " \"postprocess_sql\": [],\n", - " \"product_summaries\": [],\n", - " \"python_preprocesing_dependent\": False\n", - " }\n", - " ],\n", - " \"sql_rename_dict\": {},\n", - " \"logging_info\": {\n", - " \"Timestamp\": int(datetime.now().timestamp())\n", - " }\n", - "}\n", - "\n", - "pipeline_name = f\"hv_ti_data_loads_sage_gms_catchments_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", - "\n", - "# TODO: For later... fix fim_version value and add model_version column. current fim_version vlaue is showing 4.5.2.11\n", + "# pipeline_input = {\n", + "# \"configuration\": \"reference\",\n", + "# \"job_type\": \"auto\",\n", + "# \"data_type\": \"channel\",\n", + "# \"keep_raw\": False,\n", + "# \"reference_time\": datetime.now().strftime('%Y-%m-%d 00:00:00'),\n", + "# \"configuration_data_flow\": {\n", + "# \"db_max_flows\": [],\n", + "# \"db_ingest_groups\": [],\n", + "# \"python_preprocessing\": []\n", + "# },\n", + "# \"pipeline_products\": [\n", + "# {\n", + "# \"product\": \"static_hand_catchments_gms_branches\",\n", + "# \"configuration\": \"reference\",\n", + "# \"product_type\": \"fim\",\n", + "# \"run\": True,\n", + "# \"fim_configs\": [\n", + "# {\n", + "# \"name\": \"catchments_gms_branches\",\n", + "# \"target_table\": \"fim_catchments.catchments_gms_branches\",\n", + "# \"fim_type\": \"hand\",\n", + "# \"sql_file\": \"catchments_gms_branches\"\n", + "# }\n", + "# ],\n", + "# \"services\": [\n", + "# \"static_hand_catchments_gms_branches_noaa\"\n", + "# ],\n", + "# \"raster_outputs\": {\n", + "# \"output_bucket\": \"\",\n", + "# \"output_raster_workspaces\": []\n", + "# },\n", + "# \"postprocess_sql\": [],\n", + "# \"product_summaries\": [],\n", + "# \"python_preprocesing_dependent\": False\n", + "# },\n", + "# {\n", + "# \"product\": \"static_hand_catchments_gms_branches_hi\",\n", + "# \"configuration\": \"reference\",\n", + "# \"product_type\": \"fim\",\n", + "# \"run\": True,\n", + "# \"fim_configs\": [\n", + "# {\n", + "# \"name\": \"catchments_gms_branches_hi\",\n", + "# \"target_table\": \"fim_catchments.catchments_gms_branches_hi\",\n", + "# \"fim_type\": \"hand\",\n", + "# \"sql_file\": \"catchments_gms_branches_hi\"\n", + "# }\n", + "# ],\n", + "# \"services\": [\n", + "# \"static_hand_catchments_gms_branches_hi_noaa\"\n", + "# ],\n", + "# \"raster_outputs\": {\n", + "# \"output_bucket\": \"\",\n", + "# \"output_raster_workspaces\": []\n", + "# },\n", + "# \"postprocess_sql\": [],\n", + "# \"product_summaries\": [],\n", + "# \"python_preprocesing_dependent\": False\n", + "# },\n", + "# {\n", + "# \"product\": \"static_hand_catchments_gms_branches_prvi\",\n", + "# \"configuration\": \"reference\",\n", + "# \"product_type\": \"fim\",\n", + "# \"run\": True,\n", + "# \"fim_configs\": [\n", + "# {\n", + "# \"name\": \"catchments_gms_branches_prvi\",\n", + "# \"target_table\": \"fim_catchments.catchments_gms_branches_prvi\",\n", + "# \"fim_type\": \"hand\",\n", + "# \"sql_file\": \"catchments_gms_branches_prvi\"\n", + "# }\n", + "# ],\n", + "# \"services\": [\n", + "# \"static_hand_catchments_gms_branches_prvi_noaa\"\n", + "# ],\n", + "# \"raster_outputs\": {\n", + "# \"output_bucket\": \"\",\n", + "# \"output_raster_workspaces\": []\n", + "# },\n", + "# \"postprocess_sql\": [],\n", + "# \"product_summaries\": [],\n", + "# \"python_preprocesing_dependent\": False\n", + "# }\n", + "# ],\n", + "# \"sql_rename_dict\": {},\n", + "# \"logging_info\": {\n", + "# \"Timestamp\": int(datetime.now().timestamp())\n", + "# }\n", + "# }\n", + "\n", + "\n", + "\n", + "# STEPFUNCTION_CLIENT.start_execution(\n", + "# stateMachineArn = PIPELINE_ARN,\n", + "# name = pipeline_name,\n", + "# input= json.dumps(pipeline_input)\n", + "# )\n", + "\n", + "# print(\"Catchments GMS Branches (Level Paths / non branch 0) load kicked off.\"\n", + "# f\" Last runtime: 24:45.150. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n", + "\n", + "\n", + "pipeline_name = f\"hv_ti_data_loads_catchments_gms_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_input = get_catchment_pipepline_input(\"gms\")\n", + "# print(pipeline_input)\n", "\n", "STEPFUNCTION_CLIENT.start_execution(\n", - " stateMachineArn = PIPELINE_ARN,\n", - " name = pipeline_name,\n", - " input= json.dumps(pipeline_input)\n", + " stateMachineArn=PIPELINE_ARN,\n", + " name=pipeline_name,\n", + " input=json.dumps(pipeline_input)\n", ")\n", "\n", "print(\"Catchments GMS Branches (Level Paths / non branch 0) load kicked off.\"\n", - " f\" Last runtime: 24:45.150. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n" + " \" Takes appx 25 mins (depending on other processess)\")\n", + "print(f\" Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - {pipeline_name}\")\n" ] }, { @@ -1071,13 +1163,2178 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, "scrolled": true, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing 1 of 1000 on page 1 - (1000 per page)\n", + "Processing 2 of 1000 on page 1 - (1000 per page)\n", + "Processing 3 of 1000 on page 1 - (1000 per page)\n", + "Processing 4 of 1000 on page 1 - (1000 per page)\n", + "Processing 5 of 1000 on page 1 - (1000 per page)\n", + "Processing 6 of 1000 on page 1 - (1000 per page)\n", + "Processing 7 of 1000 on page 1 - (1000 per page)\n", + "Processing 8 of 1000 on page 1 - (1000 per page)\n", + "Processing 9 of 1000 on page 1 - (1000 per page)\n", + "Processing 10 of 1000 on page 1 - (1000 per page)\n", + "Processing 11 of 1000 on page 1 - (1000 per page)\n", + "Processing 12 of 1000 on page 1 - (1000 per page)\n", + "Processing 13 of 1000 on page 1 - (1000 per page)\n", + "Processing 14 of 1000 on page 1 - (1000 per page)\n", + "Processing 15 of 1000 on page 1 - (1000 per page)\n", + "Processing 16 of 1000 on page 1 - (1000 per page)\n", + "Processing 17 of 1000 on page 1 - (1000 per page)\n", + "Processing 18 of 1000 on page 1 - (1000 per page)\n", + "Processing 19 of 1000 on page 1 - (1000 per page)\n", + "Processing 20 of 1000 on page 1 - (1000 per page)\n", + "Processing 21 of 1000 on page 1 - (1000 per page)\n", + "Processing 22 of 1000 on page 1 - (1000 per page)\n", + "Processing 23 of 1000 on page 1 - (1000 per page)\n", + "Processing 24 of 1000 on page 1 - (1000 per page)\n", + "Processing 25 of 1000 on page 1 - (1000 per page)\n", + "Processing 26 of 1000 on page 1 - (1000 per page)\n", + "Processing 27 of 1000 on page 1 - (1000 per page)\n", + "Processing 28 of 1000 on page 1 - (1000 per page)\n", + "Processing 29 of 1000 on page 1 - (1000 per page)\n", + "Processing 30 of 1000 on page 1 - (1000 per page)\n", + "Processing 31 of 1000 on page 1 - (1000 per page)\n", + "Processing 32 of 1000 on page 1 - (1000 per page)\n", + "Processing 33 of 1000 on page 1 - (1000 per page)\n", + "Processing 34 of 1000 on page 1 - (1000 per page)\n", + "Processing 35 of 1000 on page 1 - (1000 per page)\n", + "Processing 36 of 1000 on page 1 - (1000 per page)\n", + "Processing 37 of 1000 on page 1 - (1000 per page)\n", + "Processing 38 of 1000 on page 1 - (1000 per page)\n", + "Processing 39 of 1000 on page 1 - (1000 per page)\n", + "Processing 40 of 1000 on page 1 - (1000 per page)\n", + "Processing 41 of 1000 on page 1 - (1000 per page)\n", + "Processing 42 of 1000 on page 1 - (1000 per page)\n", + "Processing 43 of 1000 on page 1 - (1000 per page)\n", + "Processing 44 of 1000 on page 1 - (1000 per page)\n", + "Processing 45 of 1000 on page 1 - (1000 per page)\n", + "Processing 46 of 1000 on page 1 - (1000 per page)\n", + "Processing 47 of 1000 on page 1 - (1000 per page)\n", + "Processing 48 of 1000 on page 1 - (1000 per page)\n", + "Processing 49 of 1000 on page 1 - (1000 per page)\n", + "Processing 50 of 1000 on page 1 - (1000 per page)\n", + "Processing 51 of 1000 on page 1 - (1000 per page)\n", + "Processing 52 of 1000 on page 1 - (1000 per page)\n", + "Processing 53 of 1000 on page 1 - (1000 per page)\n", + "Processing 54 of 1000 on page 1 - (1000 per page)\n", + "Processing 55 of 1000 on page 1 - (1000 per page)\n", + "Processing 56 of 1000 on page 1 - (1000 per page)\n", + "Processing 57 of 1000 on page 1 - (1000 per page)\n", + "Processing 58 of 1000 on page 1 - (1000 per page)\n", + "Processing 59 of 1000 on page 1 - (1000 per page)\n", + "Processing 60 of 1000 on page 1 - (1000 per page)\n", + "Processing 61 of 1000 on page 1 - (1000 per page)\n", + "Processing 62 of 1000 on page 1 - (1000 per page)\n", + "Processing 63 of 1000 on page 1 - (1000 per page)\n", + "Processing 64 of 1000 on page 1 - (1000 per page)\n", + "Processing 65 of 1000 on page 1 - (1000 per page)\n", + "Processing 66 of 1000 on page 1 - (1000 per page)\n", + "Processing 67 of 1000 on page 1 - (1000 per page)\n", + "Processing 68 of 1000 on page 1 - (1000 per page)\n", + "Processing 69 of 1000 on page 1 - (1000 per page)\n", + "Processing 70 of 1000 on page 1 - (1000 per page)\n", + "Processing 71 of 1000 on page 1 - (1000 per page)\n", + "Processing 72 of 1000 on page 1 - (1000 per page)\n", + "Processing 73 of 1000 on page 1 - (1000 per page)\n", + "Processing 74 of 1000 on page 1 - (1000 per page)\n", + "Processing 75 of 1000 on page 1 - (1000 per page)\n", + "Processing 76 of 1000 on page 1 - (1000 per page)\n", + "Processing 77 of 1000 on page 1 - (1000 per page)\n", + "Processing 78 of 1000 on page 1 - (1000 per page)\n", + "Processing 79 of 1000 on page 1 - (1000 per page)\n", + "Processing 80 of 1000 on page 1 - (1000 per page)\n", + "Processing 81 of 1000 on page 1 - (1000 per page)\n", + "Processing 82 of 1000 on page 1 - (1000 per page)\n", + "Processing 83 of 1000 on page 1 - (1000 per page)\n", + "Processing 84 of 1000 on page 1 - (1000 per page)\n", + "Processing 85 of 1000 on page 1 - (1000 per page)\n", + "Processing 86 of 1000 on page 1 - (1000 per page)\n", + "Processing 87 of 1000 on page 1 - (1000 per page)\n", + "Processing 88 of 1000 on page 1 - (1000 per page)\n", + "Processing 89 of 1000 on page 1 - (1000 per page)\n", + "Processing 90 of 1000 on page 1 - (1000 per page)\n", + "Processing 91 of 1000 on page 1 - (1000 per page)\n", + "Processing 92 of 1000 on page 1 - (1000 per page)\n", + "Processing 93 of 1000 on page 1 - (1000 per page)\n", + "Processing 94 of 1000 on page 1 - (1000 per page)\n", + "Processing 95 of 1000 on page 1 - (1000 per page)\n", + "Processing 96 of 1000 on page 1 - (1000 per page)\n", + "Processing 97 of 1000 on page 1 - (1000 per page)\n", + "Processing 98 of 1000 on page 1 - (1000 per page)\n", + "Processing 99 of 1000 on page 1 - (1000 per page)\n", + "Processing 100 of 1000 on page 1 - (1000 per page)\n", + "Processing 101 of 1000 on page 1 - (1000 per page)\n", + "Processing 102 of 1000 on page 1 - (1000 per page)\n", + "Processing 103 of 1000 on page 1 - (1000 per page)\n", + "Processing 104 of 1000 on page 1 - (1000 per page)\n", + "Processing 105 of 1000 on page 1 - (1000 per page)\n", + "Processing 106 of 1000 on page 1 - (1000 per page)\n", + "Processing 107 of 1000 on page 1 - (1000 per page)\n", + "Processing 108 of 1000 on page 1 - (1000 per page)\n", + "Processing 109 of 1000 on page 1 - (1000 per page)\n", + "Processing 110 of 1000 on page 1 - (1000 per page)\n", + "Processing 111 of 1000 on page 1 - (1000 per page)\n", + "Processing 112 of 1000 on page 1 - (1000 per page)\n", + "Processing 113 of 1000 on page 1 - (1000 per page)\n", + "Processing 114 of 1000 on page 1 - (1000 per page)\n", + "Processing 115 of 1000 on page 1 - (1000 per page)\n", + "Processing 116 of 1000 on page 1 - (1000 per page)\n", + "Processing 117 of 1000 on page 1 - (1000 per page)\n", + "Processing 118 of 1000 on page 1 - (1000 per page)\n", + "Processing 119 of 1000 on page 1 - (1000 per page)\n", + "Processing 120 of 1000 on page 1 - (1000 per page)\n", + "Processing 121 of 1000 on page 1 - (1000 per page)\n", + "Processing 122 of 1000 on page 1 - (1000 per page)\n", + "Processing 123 of 1000 on page 1 - (1000 per page)\n", + "Processing 124 of 1000 on page 1 - (1000 per page)\n", + "Processing 125 of 1000 on page 1 - (1000 per page)\n", + "Processing 126 of 1000 on page 1 - (1000 per page)\n", + "Processing 127 of 1000 on page 1 - (1000 per page)\n", + "Processing 128 of 1000 on page 1 - (1000 per page)\n", + "Processing 129 of 1000 on page 1 - (1000 per page)\n", + "Processing 130 of 1000 on page 1 - (1000 per page)\n", + "Processing 131 of 1000 on page 1 - (1000 per page)\n", + "Processing 132 of 1000 on page 1 - (1000 per page)\n", + "Processing 133 of 1000 on page 1 - (1000 per page)\n", + "Processing 134 of 1000 on page 1 - (1000 per page)\n", + "Processing 135 of 1000 on page 1 - (1000 per page)\n", + "Processing 136 of 1000 on page 1 - (1000 per page)\n", + "Processing 137 of 1000 on page 1 - (1000 per page)\n", + "Processing 138 of 1000 on page 1 - (1000 per page)\n", + "Processing 139 of 1000 on page 1 - (1000 per page)\n", + "Processing 140 of 1000 on page 1 - (1000 per page)\n", + "Processing 141 of 1000 on page 1 - (1000 per page)\n", + "Processing 142 of 1000 on page 1 - (1000 per page)\n", + "Processing 143 of 1000 on page 1 - (1000 per page)\n", + "Processing 144 of 1000 on page 1 - (1000 per page)\n", + "Processing 145 of 1000 on page 1 - (1000 per page)\n", + "Processing 146 of 1000 on page 1 - (1000 per page)\n", + "Processing 147 of 1000 on page 1 - (1000 per page)\n", + "Processing 148 of 1000 on page 1 - (1000 per page)\n", + "Processing 149 of 1000 on page 1 - (1000 per page)\n", + "Processing 150 of 1000 on page 1 - (1000 per page)\n", + "Processing 151 of 1000 on page 1 - (1000 per page)\n", + "Processing 152 of 1000 on page 1 - (1000 per page)\n", + "Processing 153 of 1000 on page 1 - (1000 per page)\n", + "Processing 154 of 1000 on page 1 - (1000 per page)\n", + "Processing 155 of 1000 on page 1 - (1000 per page)\n", + "Processing 156 of 1000 on page 1 - (1000 per page)\n", + "Processing 157 of 1000 on page 1 - (1000 per page)\n", + "Processing 158 of 1000 on page 1 - (1000 per page)\n", + "Processing 159 of 1000 on page 1 - (1000 per page)\n", + "Processing 160 of 1000 on page 1 - (1000 per page)\n", + "Processing 161 of 1000 on page 1 - (1000 per page)\n", + "Processing 162 of 1000 on page 1 - (1000 per page)\n", + "Processing 163 of 1000 on page 1 - (1000 per page)\n", + "Processing 164 of 1000 on page 1 - (1000 per page)\n", + "Processing 165 of 1000 on page 1 - (1000 per page)\n", + "Processing 166 of 1000 on page 1 - (1000 per page)\n", + "Processing 167 of 1000 on page 1 - (1000 per page)\n", + "Processing 168 of 1000 on page 1 - (1000 per page)\n", + "Processing 169 of 1000 on page 1 - (1000 per page)\n", + "Processing 170 of 1000 on page 1 - (1000 per page)\n", + "Processing 171 of 1000 on page 1 - (1000 per page)\n", + "Processing 172 of 1000 on page 1 - (1000 per page)\n", + "Processing 173 of 1000 on page 1 - (1000 per page)\n", + "Processing 174 of 1000 on page 1 - (1000 per page)\n", + "Processing 175 of 1000 on page 1 - (1000 per page)\n", + "Processing 176 of 1000 on page 1 - (1000 per page)\n", + "Processing 177 of 1000 on page 1 - (1000 per page)\n", + "Processing 178 of 1000 on page 1 - (1000 per page)\n", + "Processing 179 of 1000 on page 1 - (1000 per page)\n", + "Processing 180 of 1000 on page 1 - (1000 per page)\n", + "Processing 181 of 1000 on page 1 - (1000 per page)\n", + "Processing 182 of 1000 on page 1 - (1000 per page)\n", + "Processing 183 of 1000 on page 1 - (1000 per page)\n", + "Processing 184 of 1000 on page 1 - (1000 per page)\n", + "Processing 185 of 1000 on page 1 - (1000 per page)\n", + "Processing 186 of 1000 on page 1 - (1000 per page)\n", + "Processing 187 of 1000 on page 1 - (1000 per page)\n", + "Processing 188 of 1000 on page 1 - (1000 per page)\n", + "Processing 189 of 1000 on page 1 - (1000 per page)\n", + "Processing 190 of 1000 on page 1 - (1000 per page)\n", + "Processing 191 of 1000 on page 1 - (1000 per page)\n", + "Processing 192 of 1000 on page 1 - (1000 per page)\n", + "Processing 193 of 1000 on page 1 - (1000 per page)\n", + "Processing 194 of 1000 on page 1 - (1000 per page)\n", + "Processing 195 of 1000 on page 1 - (1000 per page)\n", + "Processing 196 of 1000 on page 1 - (1000 per page)\n", + "Processing 197 of 1000 on page 1 - (1000 per page)\n", + "Processing 198 of 1000 on page 1 - (1000 per page)\n", + "Processing 199 of 1000 on page 1 - (1000 per page)\n", + "Processing 200 of 1000 on page 1 - (1000 per page)\n", + "Processing 201 of 1000 on page 1 - (1000 per page)\n", + "Processing 202 of 1000 on page 1 - (1000 per page)\n", + "Processing 203 of 1000 on page 1 - (1000 per page)\n", + "Processing 204 of 1000 on page 1 - (1000 per page)\n", + "Processing 205 of 1000 on page 1 - (1000 per page)\n", + "Processing 206 of 1000 on page 1 - (1000 per page)\n", + "Processing 207 of 1000 on page 1 - (1000 per page)\n", + "Processing 208 of 1000 on page 1 - (1000 per page)\n", + "Processing 209 of 1000 on page 1 - (1000 per page)\n", + "Processing 210 of 1000 on page 1 - (1000 per page)\n", + "Processing 211 of 1000 on page 1 - (1000 per page)\n", + "Processing 212 of 1000 on page 1 - (1000 per page)\n", + "Processing 213 of 1000 on page 1 - (1000 per page)\n", + "Processing 214 of 1000 on page 1 - (1000 per page)\n", + "Processing 215 of 1000 on page 1 - (1000 per page)\n", + "Processing 216 of 1000 on page 1 - (1000 per page)\n", + "Processing 217 of 1000 on page 1 - (1000 per page)\n", + "Processing 218 of 1000 on page 1 - (1000 per page)\n", + "Processing 219 of 1000 on page 1 - (1000 per page)\n", + "Processing 220 of 1000 on page 1 - (1000 per page)\n", + "Processing 221 of 1000 on page 1 - (1000 per page)\n", + "Processing 222 of 1000 on page 1 - (1000 per page)\n", + "Processing 223 of 1000 on page 1 - (1000 per page)\n", + "Processing 224 of 1000 on page 1 - (1000 per page)\n", + "Processing 225 of 1000 on page 1 - (1000 per page)\n", + "Processing 226 of 1000 on page 1 - (1000 per page)\n", + "Processing 227 of 1000 on page 1 - (1000 per page)\n", + "Processing 228 of 1000 on page 1 - (1000 per page)\n", + "Processing 229 of 1000 on page 1 - (1000 per page)\n", + "Processing 230 of 1000 on page 1 - (1000 per page)\n", + "Processing 231 of 1000 on page 1 - (1000 per page)\n", + "Processing 232 of 1000 on page 1 - (1000 per page)\n", + "Processing 233 of 1000 on page 1 - (1000 per page)\n", + "Processing 234 of 1000 on page 1 - (1000 per page)\n", + "Processing 235 of 1000 on page 1 - (1000 per page)\n", + "Processing 236 of 1000 on page 1 - (1000 per page)\n", + "Processing 237 of 1000 on page 1 - (1000 per page)\n", + "Processing 238 of 1000 on page 1 - (1000 per page)\n", + "Processing 239 of 1000 on page 1 - (1000 per page)\n", + "Processing 240 of 1000 on page 1 - (1000 per page)\n", + "Processing 241 of 1000 on page 1 - (1000 per page)\n", + "Processing 242 of 1000 on page 1 - (1000 per page)\n", + "Processing 243 of 1000 on page 1 - (1000 per page)\n", + "Processing 244 of 1000 on page 1 - (1000 per page)\n", + "Processing 245 of 1000 on page 1 - (1000 per page)\n", + "Processing 246 of 1000 on page 1 - (1000 per page)\n", + "Processing 247 of 1000 on page 1 - (1000 per page)\n", + "Processing 248 of 1000 on page 1 - (1000 per page)\n", + "Processing 249 of 1000 on page 1 - (1000 per page)\n", + "Processing 250 of 1000 on page 1 - (1000 per page)\n", + "Processing 251 of 1000 on page 1 - (1000 per page)\n", + "Processing 252 of 1000 on page 1 - (1000 per page)\n", + "Processing 253 of 1000 on page 1 - (1000 per page)\n", + "Processing 254 of 1000 on page 1 - (1000 per page)\n", + "Processing 255 of 1000 on page 1 - (1000 per page)\n", + "Processing 256 of 1000 on page 1 - (1000 per page)\n", + "Processing 257 of 1000 on page 1 - (1000 per page)\n", + "Processing 258 of 1000 on page 1 - (1000 per page)\n", + "Processing 259 of 1000 on page 1 - (1000 per page)\n", + "Processing 260 of 1000 on page 1 - (1000 per page)\n", + "Processing 261 of 1000 on page 1 - (1000 per page)\n", + "Processing 262 of 1000 on page 1 - (1000 per page)\n", + "Processing 263 of 1000 on page 1 - (1000 per page)\n", + "Processing 264 of 1000 on page 1 - (1000 per page)\n", + "Processing 265 of 1000 on page 1 - (1000 per page)\n", + "Processing 266 of 1000 on page 1 - (1000 per page)\n", + "Processing 267 of 1000 on page 1 - (1000 per page)\n", + "Processing 268 of 1000 on page 1 - (1000 per page)\n", + "Processing 269 of 1000 on page 1 - (1000 per page)\n", + "Processing 270 of 1000 on page 1 - (1000 per page)\n", + "Processing 271 of 1000 on page 1 - (1000 per page)\n", + "Processing 272 of 1000 on page 1 - (1000 per page)\n", + "Processing 273 of 1000 on page 1 - (1000 per page)\n", + "Processing 274 of 1000 on page 1 - (1000 per page)\n", + "Processing 275 of 1000 on page 1 - (1000 per page)\n", + "Processing 276 of 1000 on page 1 - (1000 per page)\n", + "Processing 277 of 1000 on page 1 - (1000 per page)\n", + "Processing 278 of 1000 on page 1 - (1000 per page)\n", + "Processing 279 of 1000 on page 1 - (1000 per page)\n", + "Processing 280 of 1000 on page 1 - (1000 per page)\n", + "Processing 281 of 1000 on page 1 - (1000 per page)\n", + "Processing 282 of 1000 on page 1 - (1000 per page)\n", + "Processing 283 of 1000 on page 1 - (1000 per page)\n", + "Processing 284 of 1000 on page 1 - (1000 per page)\n", + "Processing 285 of 1000 on page 1 - (1000 per page)\n", + "Processing 286 of 1000 on page 1 - (1000 per page)\n", + "Processing 287 of 1000 on page 1 - (1000 per page)\n", + "Processing 288 of 1000 on page 1 - (1000 per page)\n", + "Processing 289 of 1000 on page 1 - (1000 per page)\n", + "Processing 290 of 1000 on page 1 - (1000 per page)\n", + "Processing 291 of 1000 on page 1 - (1000 per page)\n", + "Processing 292 of 1000 on page 1 - (1000 per page)\n", + "Processing 293 of 1000 on page 1 - (1000 per page)\n", + "Processing 294 of 1000 on page 1 - (1000 per page)\n", + "Processing 295 of 1000 on page 1 - (1000 per page)\n", + "Processing 296 of 1000 on page 1 - (1000 per page)\n", + "Processing 297 of 1000 on page 1 - (1000 per page)\n", + "Processing 298 of 1000 on page 1 - (1000 per page)\n", + "Processing 299 of 1000 on page 1 - (1000 per page)\n", + "Processing 300 of 1000 on page 1 - (1000 per page)\n", + "Processing 301 of 1000 on page 1 - (1000 per page)\n", + "Processing 302 of 1000 on page 1 - (1000 per page)\n", + "Processing 303 of 1000 on page 1 - (1000 per page)\n", + "Processing 304 of 1000 on page 1 - (1000 per page)\n", + "Processing 305 of 1000 on page 1 - (1000 per page)\n", + "Processing 306 of 1000 on page 1 - (1000 per page)\n", + "Processing 307 of 1000 on page 1 - (1000 per page)\n", + "Processing 308 of 1000 on page 1 - (1000 per page)\n", + "Processing 309 of 1000 on page 1 - (1000 per page)\n", + "Processing 310 of 1000 on page 1 - (1000 per page)\n", + "Processing 311 of 1000 on page 1 - (1000 per page)\n", + "Processing 312 of 1000 on page 1 - (1000 per page)\n", + "Processing 313 of 1000 on page 1 - (1000 per page)\n", + "Processing 314 of 1000 on page 1 - (1000 per page)\n", + "Processing 315 of 1000 on page 1 - (1000 per page)\n", + "Processing 316 of 1000 on page 1 - (1000 per page)\n", + "Processing 317 of 1000 on page 1 - (1000 per page)\n", + "Processing 318 of 1000 on page 1 - (1000 per page)\n", + "Processing 319 of 1000 on page 1 - (1000 per page)\n", + "Processing 320 of 1000 on page 1 - (1000 per page)\n", + "Processing 321 of 1000 on page 1 - (1000 per page)\n", + "Processing 322 of 1000 on page 1 - (1000 per page)\n", + "Processing 323 of 1000 on page 1 - (1000 per page)\n", + "Processing 324 of 1000 on page 1 - (1000 per page)\n", + "Processing 325 of 1000 on page 1 - (1000 per page)\n", + "Processing 326 of 1000 on page 1 - (1000 per page)\n", + "Processing 327 of 1000 on page 1 - (1000 per page)\n", + "Processing 328 of 1000 on page 1 - (1000 per page)\n", + "Processing 329 of 1000 on page 1 - (1000 per page)\n", + "Processing 330 of 1000 on page 1 - (1000 per page)\n", + "Processing 331 of 1000 on page 1 - (1000 per page)\n", + "Processing 332 of 1000 on page 1 - (1000 per page)\n", + "Processing 333 of 1000 on page 1 - (1000 per page)\n", + "Processing 334 of 1000 on page 1 - (1000 per page)\n", + "Processing 335 of 1000 on page 1 - (1000 per page)\n", + "Processing 336 of 1000 on page 1 - (1000 per page)\n", + "Processing 337 of 1000 on page 1 - (1000 per page)\n", + "Processing 338 of 1000 on page 1 - (1000 per page)\n", + "Processing 339 of 1000 on page 1 - (1000 per page)\n", + "Processing 340 of 1000 on page 1 - (1000 per page)\n", + "Processing 341 of 1000 on page 1 - (1000 per page)\n", + "Processing 342 of 1000 on page 1 - (1000 per page)\n", + "Processing 343 of 1000 on page 1 - (1000 per page)\n", + "Processing 344 of 1000 on page 1 - (1000 per page)\n", + "Processing 345 of 1000 on page 1 - (1000 per page)\n", + "Processing 346 of 1000 on page 1 - (1000 per page)\n", + "Processing 347 of 1000 on page 1 - (1000 per page)\n", + "Processing 348 of 1000 on page 1 - (1000 per page)\n", + "Processing 349 of 1000 on page 1 - (1000 per page)\n", + "Processing 350 of 1000 on page 1 - (1000 per page)\n", + "Processing 351 of 1000 on page 1 - (1000 per page)\n", + "Processing 352 of 1000 on page 1 - (1000 per page)\n", + "Processing 353 of 1000 on page 1 - (1000 per page)\n", + "Processing 354 of 1000 on page 1 - (1000 per page)\n", + "Processing 355 of 1000 on page 1 - (1000 per page)\n", + "Processing 356 of 1000 on page 1 - (1000 per page)\n", + "Processing 357 of 1000 on page 1 - (1000 per page)\n", + "Processing 358 of 1000 on page 1 - (1000 per page)\n", + "Processing 359 of 1000 on page 1 - (1000 per page)\n", + "Processing 360 of 1000 on page 1 - (1000 per page)\n", + "Processing 361 of 1000 on page 1 - (1000 per page)\n", + "Processing 362 of 1000 on page 1 - (1000 per page)\n", + "Processing 363 of 1000 on page 1 - (1000 per page)\n", + "Processing 364 of 1000 on page 1 - (1000 per page)\n", + "Processing 365 of 1000 on page 1 - (1000 per page)\n", + "Processing 366 of 1000 on page 1 - (1000 per page)\n", + "Processing 367 of 1000 on page 1 - (1000 per page)\n", + "Processing 368 of 1000 on page 1 - (1000 per page)\n", + "Processing 369 of 1000 on page 1 - (1000 per page)\n", + "Processing 370 of 1000 on page 1 - (1000 per page)\n", + "Processing 371 of 1000 on page 1 - (1000 per page)\n", + "Processing 372 of 1000 on page 1 - (1000 per page)\n", + "Processing 373 of 1000 on page 1 - (1000 per page)\n", + "Processing 374 of 1000 on page 1 - (1000 per page)\n", + "Processing 375 of 1000 on page 1 - (1000 per page)\n", + "Processing 376 of 1000 on page 1 - (1000 per page)\n", + "Processing 377 of 1000 on page 1 - (1000 per page)\n", + "Processing 378 of 1000 on page 1 - (1000 per page)\n", + "Processing 379 of 1000 on page 1 - (1000 per page)\n", + "Processing 380 of 1000 on page 1 - (1000 per page)\n", + "Processing 381 of 1000 on page 1 - (1000 per page)\n", + "Processing 382 of 1000 on page 1 - (1000 per page)\n", + "Processing 383 of 1000 on page 1 - (1000 per page)\n", + "Processing 384 of 1000 on page 1 - (1000 per page)\n", + "Processing 385 of 1000 on page 1 - (1000 per page)\n", + "Processing 386 of 1000 on page 1 - (1000 per page)\n", + "Processing 387 of 1000 on page 1 - (1000 per page)\n", + "Processing 388 of 1000 on page 1 - (1000 per page)\n", + "Processing 389 of 1000 on page 1 - (1000 per page)\n", + "Processing 390 of 1000 on page 1 - (1000 per page)\n", + "Processing 391 of 1000 on page 1 - (1000 per page)\n", + "Processing 392 of 1000 on page 1 - (1000 per page)\n", + "Processing 393 of 1000 on page 1 - (1000 per page)\n", + "Processing 394 of 1000 on page 1 - (1000 per page)\n", + "Processing 395 of 1000 on page 1 - (1000 per page)\n", + "Processing 396 of 1000 on page 1 - (1000 per page)\n", + "Processing 397 of 1000 on page 1 - (1000 per page)\n", + "Processing 398 of 1000 on page 1 - (1000 per page)\n", + "Processing 399 of 1000 on page 1 - (1000 per page)\n", + "Processing 400 of 1000 on page 1 - (1000 per page)\n", + "Processing 401 of 1000 on page 1 - (1000 per page)\n", + "Processing 402 of 1000 on page 1 - (1000 per page)\n", + "Processing 403 of 1000 on page 1 - (1000 per page)\n", + "Processing 404 of 1000 on page 1 - (1000 per page)\n", + "Processing 405 of 1000 on page 1 - (1000 per page)\n", + "Processing 406 of 1000 on page 1 - (1000 per page)\n", + "Processing 407 of 1000 on page 1 - (1000 per page)\n", + "Processing 408 of 1000 on page 1 - (1000 per page)\n", + "Processing 409 of 1000 on page 1 - (1000 per page)\n", + "Processing 410 of 1000 on page 1 - (1000 per page)\n", + "Processing 411 of 1000 on page 1 - (1000 per page)\n", + "Processing 412 of 1000 on page 1 - (1000 per page)\n", + "Processing 413 of 1000 on page 1 - (1000 per page)\n", + "Processing 414 of 1000 on page 1 - (1000 per page)\n", + "Processing 415 of 1000 on page 1 - (1000 per page)\n", + "Processing 416 of 1000 on page 1 - (1000 per page)\n", + "Processing 417 of 1000 on page 1 - (1000 per page)\n", + "Processing 418 of 1000 on page 1 - (1000 per page)\n", + "Processing 419 of 1000 on page 1 - (1000 per page)\n", + "Processing 420 of 1000 on page 1 - (1000 per page)\n", + "Processing 421 of 1000 on page 1 - (1000 per page)\n", + "Processing 422 of 1000 on page 1 - (1000 per page)\n", + "Processing 423 of 1000 on page 1 - (1000 per page)\n", + "Processing 424 of 1000 on page 1 - (1000 per page)\n", + "Processing 425 of 1000 on page 1 - (1000 per page)\n", + "Processing 426 of 1000 on page 1 - (1000 per page)\n", + "Processing 427 of 1000 on page 1 - (1000 per page)\n", + "Processing 428 of 1000 on page 1 - (1000 per page)\n", + "Processing 429 of 1000 on page 1 - (1000 per page)\n", + "Processing 430 of 1000 on page 1 - (1000 per page)\n", + "Processing 431 of 1000 on page 1 - (1000 per page)\n", + "Processing 432 of 1000 on page 1 - (1000 per page)\n", + "Processing 433 of 1000 on page 1 - (1000 per page)\n", + "Processing 434 of 1000 on page 1 - (1000 per page)\n", + "Processing 435 of 1000 on page 1 - (1000 per page)\n", + "Processing 436 of 1000 on page 1 - (1000 per page)\n", + "Processing 437 of 1000 on page 1 - (1000 per page)\n", + "Processing 438 of 1000 on page 1 - (1000 per page)\n", + "Processing 439 of 1000 on page 1 - (1000 per page)\n", + "Processing 440 of 1000 on page 1 - (1000 per page)\n", + "Processing 441 of 1000 on page 1 - (1000 per page)\n", + "Processing 442 of 1000 on page 1 - (1000 per page)\n", + "Processing 443 of 1000 on page 1 - (1000 per page)\n", + "Processing 444 of 1000 on page 1 - (1000 per page)\n", + "Processing 445 of 1000 on page 1 - (1000 per page)\n", + "Processing 446 of 1000 on page 1 - (1000 per page)\n", + "Processing 447 of 1000 on page 1 - (1000 per page)\n", + "Processing 448 of 1000 on page 1 - (1000 per page)\n", + "Processing 449 of 1000 on page 1 - (1000 per page)\n", + "Processing 450 of 1000 on page 1 - (1000 per page)\n", + "Processing 451 of 1000 on page 1 - (1000 per page)\n", + "Processing 452 of 1000 on page 1 - (1000 per page)\n", + "Processing 453 of 1000 on page 1 - (1000 per page)\n", + "Processing 454 of 1000 on page 1 - (1000 per page)\n", + "Processing 455 of 1000 on page 1 - (1000 per page)\n", + "Processing 456 of 1000 on page 1 - (1000 per page)\n", + "Processing 457 of 1000 on page 1 - (1000 per page)\n", + "Processing 458 of 1000 on page 1 - (1000 per page)\n", + "Processing 459 of 1000 on page 1 - (1000 per page)\n", + "Processing 460 of 1000 on page 1 - (1000 per page)\n", + "Processing 461 of 1000 on page 1 - (1000 per page)\n", + "Processing 462 of 1000 on page 1 - (1000 per page)\n", + "Processing 463 of 1000 on page 1 - (1000 per page)\n", + "Processing 464 of 1000 on page 1 - (1000 per page)\n", + "Processing 465 of 1000 on page 1 - (1000 per page)\n", + "Processing 466 of 1000 on page 1 - (1000 per page)\n", + "Processing 467 of 1000 on page 1 - (1000 per page)\n", + "Processing 468 of 1000 on page 1 - (1000 per page)\n", + "Processing 469 of 1000 on page 1 - (1000 per page)\n", + "Processing 470 of 1000 on page 1 - (1000 per page)\n", + "Processing 471 of 1000 on page 1 - (1000 per page)\n", + "Processing 472 of 1000 on page 1 - (1000 per page)\n", + "Processing 473 of 1000 on page 1 - (1000 per page)\n", + "Processing 474 of 1000 on page 1 - (1000 per page)\n", + "Processing 475 of 1000 on page 1 - (1000 per page)\n", + "Processing 476 of 1000 on page 1 - (1000 per page)\n", + "Processing 477 of 1000 on page 1 - (1000 per page)\n", + "Processing 478 of 1000 on page 1 - (1000 per page)\n", + "Processing 479 of 1000 on page 1 - (1000 per page)\n", + "Processing 480 of 1000 on page 1 - (1000 per page)\n", + "Processing 481 of 1000 on page 1 - (1000 per page)\n", + "Processing 482 of 1000 on page 1 - (1000 per page)\n", + "Processing 483 of 1000 on page 1 - (1000 per page)\n", + "Processing 484 of 1000 on page 1 - (1000 per page)\n", + "Processing 485 of 1000 on page 1 - (1000 per page)\n", + "Processing 486 of 1000 on page 1 - (1000 per page)\n", + "Processing 487 of 1000 on page 1 - (1000 per page)\n", + "Processing 488 of 1000 on page 1 - (1000 per page)\n", + "Processing 489 of 1000 on page 1 - (1000 per page)\n", + "Processing 490 of 1000 on page 1 - (1000 per page)\n", + "Processing 491 of 1000 on page 1 - (1000 per page)\n", + "Processing 492 of 1000 on page 1 - (1000 per page)\n", + "Processing 493 of 1000 on page 1 - (1000 per page)\n", + "Processing 494 of 1000 on page 1 - (1000 per page)\n", + "Processing 495 of 1000 on page 1 - (1000 per page)\n", + "Processing 496 of 1000 on page 1 - (1000 per page)\n", + "Processing 497 of 1000 on page 1 - (1000 per page)\n", + "Processing 498 of 1000 on page 1 - (1000 per page)\n", + "Processing 499 of 1000 on page 1 - (1000 per page)\n", + "Processing 500 of 1000 on page 1 - (1000 per page)\n", + "Processing 501 of 1000 on page 1 - (1000 per page)\n", + "Processing 502 of 1000 on page 1 - (1000 per page)\n", + "Processing 503 of 1000 on page 1 - (1000 per page)\n", + "Processing 504 of 1000 on page 1 - (1000 per page)\n", + "Processing 505 of 1000 on page 1 - (1000 per page)\n", + "Processing 506 of 1000 on page 1 - (1000 per page)\n", + "Processing 507 of 1000 on page 1 - (1000 per page)\n", + "Processing 508 of 1000 on page 1 - (1000 per page)\n", + "Processing 509 of 1000 on page 1 - (1000 per page)\n", + "Processing 510 of 1000 on page 1 - (1000 per page)\n", + "Processing 511 of 1000 on page 1 - (1000 per page)\n", + "Processing 512 of 1000 on page 1 - (1000 per page)\n", + "Processing 513 of 1000 on page 1 - (1000 per page)\n", + "Processing 514 of 1000 on page 1 - (1000 per page)\n", + "Processing 515 of 1000 on page 1 - (1000 per page)\n", + "Processing 516 of 1000 on page 1 - (1000 per page)\n", + "Processing 517 of 1000 on page 1 - (1000 per page)\n", + "Processing 518 of 1000 on page 1 - (1000 per page)\n", + "Processing 519 of 1000 on page 1 - (1000 per page)\n", + "Processing 520 of 1000 on page 1 - (1000 per page)\n", + "Processing 521 of 1000 on page 1 - (1000 per page)\n", + "Processing 522 of 1000 on page 1 - (1000 per page)\n", + "Processing 523 of 1000 on page 1 - (1000 per page)\n", + "Processing 524 of 1000 on page 1 - (1000 per page)\n", + "Processing 525 of 1000 on page 1 - (1000 per page)\n", + "Processing 526 of 1000 on page 1 - (1000 per page)\n", + "Processing 527 of 1000 on page 1 - (1000 per page)\n", + "Processing 528 of 1000 on page 1 - (1000 per page)\n", + "Processing 529 of 1000 on page 1 - (1000 per page)\n", + "Processing 530 of 1000 on page 1 - (1000 per page)\n", + "Processing 531 of 1000 on page 1 - (1000 per page)\n", + "Processing 532 of 1000 on page 1 - (1000 per page)\n", + "Processing 533 of 1000 on page 1 - (1000 per page)\n", + "Processing 534 of 1000 on page 1 - (1000 per page)\n", + "Processing 535 of 1000 on page 1 - (1000 per page)\n", + "Processing 536 of 1000 on page 1 - (1000 per page)\n", + "Processing 537 of 1000 on page 1 - (1000 per page)\n", + "Processing 538 of 1000 on page 1 - (1000 per page)\n", + "Processing 539 of 1000 on page 1 - (1000 per page)\n", + "Processing 540 of 1000 on page 1 - (1000 per page)\n", + "Processing 541 of 1000 on page 1 - (1000 per page)\n", + "Processing 542 of 1000 on page 1 - (1000 per page)\n", + "Processing 543 of 1000 on page 1 - (1000 per page)\n", + "Processing 544 of 1000 on page 1 - (1000 per page)\n", + "Processing 545 of 1000 on page 1 - (1000 per page)\n", + "Processing 546 of 1000 on page 1 - (1000 per page)\n", + "Processing 547 of 1000 on page 1 - (1000 per page)\n", + "Processing 548 of 1000 on page 1 - (1000 per page)\n", + "Processing 549 of 1000 on page 1 - (1000 per page)\n", + "Processing 550 of 1000 on page 1 - (1000 per page)\n", + "Processing 551 of 1000 on page 1 - (1000 per page)\n", + "Processing 552 of 1000 on page 1 - (1000 per page)\n", + "Processing 553 of 1000 on page 1 - (1000 per page)\n", + "Processing 554 of 1000 on page 1 - (1000 per page)\n", + "Processing 555 of 1000 on page 1 - (1000 per page)\n", + "Processing 556 of 1000 on page 1 - (1000 per page)\n", + "Processing 557 of 1000 on page 1 - (1000 per page)\n", + "Processing 558 of 1000 on page 1 - (1000 per page)\n", + "Processing 559 of 1000 on page 1 - (1000 per page)\n", + "Processing 560 of 1000 on page 1 - (1000 per page)\n", + "Processing 561 of 1000 on page 1 - (1000 per page)\n", + "Processing 562 of 1000 on page 1 - (1000 per page)\n", + "Processing 563 of 1000 on page 1 - (1000 per page)\n", + "Processing 564 of 1000 on page 1 - (1000 per page)\n", + "Processing 565 of 1000 on page 1 - (1000 per page)\n", + "Processing 566 of 1000 on page 1 - (1000 per page)\n", + "Processing 567 of 1000 on page 1 - (1000 per page)\n", + "Processing 568 of 1000 on page 1 - (1000 per page)\n", + "Processing 569 of 1000 on page 1 - (1000 per page)\n", + "Processing 570 of 1000 on page 1 - (1000 per page)\n", + "Processing 571 of 1000 on page 1 - (1000 per page)\n", + "Processing 572 of 1000 on page 1 - (1000 per page)\n", + "Processing 573 of 1000 on page 1 - (1000 per page)\n", + "Processing 574 of 1000 on page 1 - (1000 per page)\n", + "Processing 575 of 1000 on page 1 - (1000 per page)\n", + "Processing 576 of 1000 on page 1 - (1000 per page)\n", + "Processing 577 of 1000 on page 1 - (1000 per page)\n", + "Processing 578 of 1000 on page 1 - (1000 per page)\n", + "Processing 579 of 1000 on page 1 - (1000 per page)\n", + "Processing 580 of 1000 on page 1 - (1000 per page)\n", + "Processing 581 of 1000 on page 1 - (1000 per page)\n", + "Processing 582 of 1000 on page 1 - (1000 per page)\n", + "Processing 583 of 1000 on page 1 - (1000 per page)\n", + "Processing 584 of 1000 on page 1 - (1000 per page)\n", + "Processing 585 of 1000 on page 1 - (1000 per page)\n", + "Processing 586 of 1000 on page 1 - (1000 per page)\n", + "Processing 587 of 1000 on page 1 - (1000 per page)\n", + "Processing 588 of 1000 on page 1 - (1000 per page)\n", + "Processing 589 of 1000 on page 1 - (1000 per page)\n", + "Processing 590 of 1000 on page 1 - (1000 per page)\n", + "Processing 591 of 1000 on page 1 - (1000 per page)\n", + "Processing 592 of 1000 on page 1 - (1000 per page)\n", + "Processing 593 of 1000 on page 1 - (1000 per page)\n", + "Processing 594 of 1000 on page 1 - (1000 per page)\n", + "Processing 595 of 1000 on page 1 - (1000 per page)\n", + "Processing 596 of 1000 on page 1 - (1000 per page)\n", + "Processing 597 of 1000 on page 1 - (1000 per page)\n", + "Processing 598 of 1000 on page 1 - (1000 per page)\n", + "Processing 599 of 1000 on page 1 - (1000 per page)\n", + "Processing 600 of 1000 on page 1 - (1000 per page)\n", + "Processing 601 of 1000 on page 1 - (1000 per page)\n", + "Processing 602 of 1000 on page 1 - (1000 per page)\n", + "Processing 603 of 1000 on page 1 - (1000 per page)\n", + "Processing 604 of 1000 on page 1 - (1000 per page)\n", + "Processing 605 of 1000 on page 1 - (1000 per page)\n", + "Processing 606 of 1000 on page 1 - (1000 per page)\n", + "Processing 607 of 1000 on page 1 - (1000 per page)\n", + "Processing 608 of 1000 on page 1 - (1000 per page)\n", + "Processing 609 of 1000 on page 1 - (1000 per page)\n", + "Processing 610 of 1000 on page 1 - (1000 per page)\n", + "Processing 611 of 1000 on page 1 - (1000 per page)\n", + "Processing 612 of 1000 on page 1 - (1000 per page)\n", + "Processing 613 of 1000 on page 1 - (1000 per page)\n", + "Processing 614 of 1000 on page 1 - (1000 per page)\n", + "Processing 615 of 1000 on page 1 - (1000 per page)\n", + "Processing 616 of 1000 on page 1 - (1000 per page)\n", + "Processing 617 of 1000 on page 1 - (1000 per page)\n", + "Processing 618 of 1000 on page 1 - (1000 per page)\n", + "Processing 619 of 1000 on page 1 - (1000 per page)\n", + "Processing 620 of 1000 on page 1 - (1000 per page)\n", + "Processing 621 of 1000 on page 1 - (1000 per page)\n", + "Processing 622 of 1000 on page 1 - (1000 per page)\n", + "Processing 623 of 1000 on page 1 - (1000 per page)\n", + "Processing 624 of 1000 on page 1 - (1000 per page)\n", + "Processing 625 of 1000 on page 1 - (1000 per page)\n", + "Processing 626 of 1000 on page 1 - (1000 per page)\n", + "Processing 627 of 1000 on page 1 - (1000 per page)\n", + "Processing 628 of 1000 on page 1 - (1000 per page)\n", + "Processing 629 of 1000 on page 1 - (1000 per page)\n", + "Processing 630 of 1000 on page 1 - (1000 per page)\n", + "Processing 631 of 1000 on page 1 - (1000 per page)\n", + "Processing 632 of 1000 on page 1 - (1000 per page)\n", + "Processing 633 of 1000 on page 1 - (1000 per page)\n", + "Processing 634 of 1000 on page 1 - (1000 per page)\n", + "Processing 635 of 1000 on page 1 - (1000 per page)\n", + "Processing 636 of 1000 on page 1 - (1000 per page)\n", + "Processing 637 of 1000 on page 1 - (1000 per page)\n", + "Processing 638 of 1000 on page 1 - (1000 per page)\n", + "Processing 639 of 1000 on page 1 - (1000 per page)\n", + "Processing 640 of 1000 on page 1 - (1000 per page)\n", + "Processing 641 of 1000 on page 1 - (1000 per page)\n", + "Processing 642 of 1000 on page 1 - (1000 per page)\n", + "Processing 643 of 1000 on page 1 - (1000 per page)\n", + "Processing 644 of 1000 on page 1 - (1000 per page)\n", + "Processing 645 of 1000 on page 1 - (1000 per page)\n", + "Processing 646 of 1000 on page 1 - (1000 per page)\n", + "Processing 647 of 1000 on page 1 - (1000 per page)\n", + "Processing 648 of 1000 on page 1 - (1000 per page)\n", + "Processing 649 of 1000 on page 1 - (1000 per page)\n", + "Processing 650 of 1000 on page 1 - (1000 per page)\n", + "Processing 651 of 1000 on page 1 - (1000 per page)\n", + "Processing 652 of 1000 on page 1 - (1000 per page)\n", + "Processing 653 of 1000 on page 1 - (1000 per page)\n", + "Processing 654 of 1000 on page 1 - (1000 per page)\n", + "Processing 655 of 1000 on page 1 - (1000 per page)\n", + "Processing 656 of 1000 on page 1 - (1000 per page)\n", + "Processing 657 of 1000 on page 1 - (1000 per page)\n", + "Processing 658 of 1000 on page 1 - (1000 per page)\n", + "Processing 659 of 1000 on page 1 - (1000 per page)\n", + "Processing 660 of 1000 on page 1 - (1000 per page)\n", + "Processing 661 of 1000 on page 1 - (1000 per page)\n", + "Processing 662 of 1000 on page 1 - (1000 per page)\n", + "Processing 663 of 1000 on page 1 - (1000 per page)\n", + "Processing 664 of 1000 on page 1 - (1000 per page)\n", + "Processing 665 of 1000 on page 1 - (1000 per page)\n", + "Processing 666 of 1000 on page 1 - (1000 per page)\n", + "Processing 667 of 1000 on page 1 - (1000 per page)\n", + "Processing 668 of 1000 on page 1 - (1000 per page)\n", + "Processing 669 of 1000 on page 1 - (1000 per page)\n", + "Processing 670 of 1000 on page 1 - (1000 per page)\n", + "Processing 671 of 1000 on page 1 - (1000 per page)\n", + "Processing 672 of 1000 on page 1 - (1000 per page)\n", + "Processing 673 of 1000 on page 1 - (1000 per page)\n", + "Processing 674 of 1000 on page 1 - (1000 per page)\n", + "Processing 675 of 1000 on page 1 - (1000 per page)\n", + "Processing 676 of 1000 on page 1 - (1000 per page)\n", + "Processing 677 of 1000 on page 1 - (1000 per page)\n", + "Processing 678 of 1000 on page 1 - (1000 per page)\n", + "Processing 679 of 1000 on page 1 - (1000 per page)\n", + "Processing 680 of 1000 on page 1 - (1000 per page)\n", + "Processing 681 of 1000 on page 1 - (1000 per page)\n", + "Processing 682 of 1000 on page 1 - (1000 per page)\n", + "Processing 683 of 1000 on page 1 - (1000 per page)\n", + "Processing 684 of 1000 on page 1 - (1000 per page)\n", + "Processing 685 of 1000 on page 1 - (1000 per page)\n", + "Processing 686 of 1000 on page 1 - (1000 per page)\n", + "Processing 687 of 1000 on page 1 - (1000 per page)\n", + "Processing 688 of 1000 on page 1 - (1000 per page)\n", + "Processing 689 of 1000 on page 1 - (1000 per page)\n", + "Processing 690 of 1000 on page 1 - (1000 per page)\n", + "Processing 691 of 1000 on page 1 - (1000 per page)\n", + "Processing 692 of 1000 on page 1 - (1000 per page)\n", + "Processing 693 of 1000 on page 1 - (1000 per page)\n", + "Processing 694 of 1000 on page 1 - (1000 per page)\n", + "Processing 695 of 1000 on page 1 - (1000 per page)\n", + "Processing 696 of 1000 on page 1 - (1000 per page)\n", + "Processing 697 of 1000 on page 1 - (1000 per page)\n", + "Processing 698 of 1000 on page 1 - (1000 per page)\n", + "Processing 699 of 1000 on page 1 - (1000 per page)\n", + "Processing 700 of 1000 on page 1 - (1000 per page)\n", + "Processing 701 of 1000 on page 1 - (1000 per page)\n", + "Processing 702 of 1000 on page 1 - (1000 per page)\n", + "Processing 703 of 1000 on page 1 - (1000 per page)\n", + "Processing 704 of 1000 on page 1 - (1000 per page)\n", + "Processing 705 of 1000 on page 1 - (1000 per page)\n", + "Processing 706 of 1000 on page 1 - (1000 per page)\n", + "Processing 707 of 1000 on page 1 - (1000 per page)\n", + "Processing 708 of 1000 on page 1 - (1000 per page)\n", + "Processing 709 of 1000 on page 1 - (1000 per page)\n", + "Processing 710 of 1000 on page 1 - (1000 per page)\n", + "Processing 711 of 1000 on page 1 - (1000 per page)\n", + "Processing 712 of 1000 on page 1 - (1000 per page)\n", + "Processing 713 of 1000 on page 1 - (1000 per page)\n", + "Processing 714 of 1000 on page 1 - (1000 per page)\n", + "Processing 715 of 1000 on page 1 - (1000 per page)\n", + "Processing 716 of 1000 on page 1 - (1000 per page)\n", + "Processing 717 of 1000 on page 1 - (1000 per page)\n", + "Processing 718 of 1000 on page 1 - (1000 per page)\n", + "Processing 719 of 1000 on page 1 - (1000 per page)\n", + "Processing 720 of 1000 on page 1 - (1000 per page)\n", + "Processing 721 of 1000 on page 1 - (1000 per page)\n", + "Processing 722 of 1000 on page 1 - (1000 per page)\n", + "Processing 723 of 1000 on page 1 - (1000 per page)\n", + "Processing 724 of 1000 on page 1 - (1000 per page)\n", + "Processing 725 of 1000 on page 1 - (1000 per page)\n", + "Processing 726 of 1000 on page 1 - (1000 per page)\n", + "Processing 727 of 1000 on page 1 - (1000 per page)\n", + "Processing 728 of 1000 on page 1 - (1000 per page)\n", + "Processing 729 of 1000 on page 1 - (1000 per page)\n", + "Processing 730 of 1000 on page 1 - (1000 per page)\n", + "Processing 731 of 1000 on page 1 - (1000 per page)\n", + "Processing 732 of 1000 on page 1 - (1000 per page)\n", + "Processing 733 of 1000 on page 1 - (1000 per page)\n", + "Processing 734 of 1000 on page 1 - (1000 per page)\n", + "Processing 735 of 1000 on page 1 - (1000 per page)\n", + "Processing 736 of 1000 on page 1 - (1000 per page)\n", + "Processing 737 of 1000 on page 1 - (1000 per page)\n", + "Processing 738 of 1000 on page 1 - (1000 per page)\n", + "Processing 739 of 1000 on page 1 - (1000 per page)\n", + "Processing 740 of 1000 on page 1 - (1000 per page)\n", + "Processing 741 of 1000 on page 1 - (1000 per page)\n", + "Processing 742 of 1000 on page 1 - (1000 per page)\n", + "Processing 743 of 1000 on page 1 - (1000 per page)\n", + "Processing 744 of 1000 on page 1 - (1000 per page)\n", + "Processing 745 of 1000 on page 1 - (1000 per page)\n", + "Processing 746 of 1000 on page 1 - (1000 per page)\n", + "Processing 747 of 1000 on page 1 - (1000 per page)\n", + "Processing 748 of 1000 on page 1 - (1000 per page)\n", + "Processing 749 of 1000 on page 1 - (1000 per page)\n", + "Processing 750 of 1000 on page 1 - (1000 per page)\n", + "Processing 751 of 1000 on page 1 - (1000 per page)\n", + "Processing 752 of 1000 on page 1 - (1000 per page)\n", + "Processing 753 of 1000 on page 1 - (1000 per page)\n", + "Processing 754 of 1000 on page 1 - (1000 per page)\n", + "Processing 755 of 1000 on page 1 - (1000 per page)\n", + "Processing 756 of 1000 on page 1 - (1000 per page)\n", + "Processing 757 of 1000 on page 1 - (1000 per page)\n", + "Processing 758 of 1000 on page 1 - (1000 per page)\n", + "Processing 759 of 1000 on page 1 - (1000 per page)\n", + "Processing 760 of 1000 on page 1 - (1000 per page)\n", + "Processing 761 of 1000 on page 1 - (1000 per page)\n", + "Processing 762 of 1000 on page 1 - (1000 per page)\n", + "Processing 763 of 1000 on page 1 - (1000 per page)\n", + "Processing 764 of 1000 on page 1 - (1000 per page)\n", + "Processing 765 of 1000 on page 1 - (1000 per page)\n", + "Processing 766 of 1000 on page 1 - (1000 per page)\n", + "Processing 767 of 1000 on page 1 - (1000 per page)\n", + "Processing 768 of 1000 on page 1 - (1000 per page)\n", + "Processing 769 of 1000 on page 1 - (1000 per page)\n", + "Processing 770 of 1000 on page 1 - (1000 per page)\n", + "Processing 771 of 1000 on page 1 - (1000 per page)\n", + "Processing 772 of 1000 on page 1 - (1000 per page)\n", + "Processing 773 of 1000 on page 1 - (1000 per page)\n", + "Processing 774 of 1000 on page 1 - (1000 per page)\n", + "Processing 775 of 1000 on page 1 - (1000 per page)\n", + "Processing 776 of 1000 on page 1 - (1000 per page)\n", + "Processing 777 of 1000 on page 1 - (1000 per page)\n", + "Processing 778 of 1000 on page 1 - (1000 per page)\n", + "Processing 779 of 1000 on page 1 - (1000 per page)\n", + "Processing 780 of 1000 on page 1 - (1000 per page)\n", + "Processing 781 of 1000 on page 1 - (1000 per page)\n", + "Processing 782 of 1000 on page 1 - (1000 per page)\n", + "Processing 783 of 1000 on page 1 - (1000 per page)\n", + "Processing 784 of 1000 on page 1 - (1000 per page)\n", + "Processing 785 of 1000 on page 1 - (1000 per page)\n", + "Processing 786 of 1000 on page 1 - (1000 per page)\n", + "Processing 787 of 1000 on page 1 - (1000 per page)\n", + "Processing 788 of 1000 on page 1 - (1000 per page)\n", + "Processing 789 of 1000 on page 1 - (1000 per page)\n", + "Processing 790 of 1000 on page 1 - (1000 per page)\n", + "Processing 791 of 1000 on page 1 - (1000 per page)\n", + "Processing 792 of 1000 on page 1 - (1000 per page)\n", + "Processing 793 of 1000 on page 1 - (1000 per page)\n", + "Processing 794 of 1000 on page 1 - (1000 per page)\n", + "Processing 795 of 1000 on page 1 - (1000 per page)\n", + "Processing 796 of 1000 on page 1 - (1000 per page)\n", + "Processing 797 of 1000 on page 1 - (1000 per page)\n", + "Processing 798 of 1000 on page 1 - (1000 per page)\n", + "Processing 799 of 1000 on page 1 - (1000 per page)\n", + "Processing 800 of 1000 on page 1 - (1000 per page)\n", + "Processing 801 of 1000 on page 1 - (1000 per page)\n", + "Processing 802 of 1000 on page 1 - (1000 per page)\n", + "Processing 803 of 1000 on page 1 - (1000 per page)\n", + "Processing 804 of 1000 on page 1 - (1000 per page)\n", + "Processing 805 of 1000 on page 1 - (1000 per page)\n", + "Processing 806 of 1000 on page 1 - (1000 per page)\n", + "Processing 807 of 1000 on page 1 - (1000 per page)\n", + "Processing 808 of 1000 on page 1 - (1000 per page)\n", + "Processing 809 of 1000 on page 1 - (1000 per page)\n", + "Processing 810 of 1000 on page 1 - (1000 per page)\n", + "Processing 811 of 1000 on page 1 - (1000 per page)\n", + "Processing 812 of 1000 on page 1 - (1000 per page)\n", + "Processing 813 of 1000 on page 1 - (1000 per page)\n", + "Processing 814 of 1000 on page 1 - (1000 per page)\n", + "Processing 815 of 1000 on page 1 - (1000 per page)\n", + "Processing 816 of 1000 on page 1 - (1000 per page)\n", + "Processing 817 of 1000 on page 1 - (1000 per page)\n", + "Processing 818 of 1000 on page 1 - (1000 per page)\n", + "Processing 819 of 1000 on page 1 - (1000 per page)\n", + "Processing 820 of 1000 on page 1 - (1000 per page)\n", + "Processing 821 of 1000 on page 1 - (1000 per page)\n", + "Processing 822 of 1000 on page 1 - (1000 per page)\n", + "Processing 823 of 1000 on page 1 - (1000 per page)\n", + "Processing 824 of 1000 on page 1 - (1000 per page)\n", + "Processing 825 of 1000 on page 1 - (1000 per page)\n", + "Processing 826 of 1000 on page 1 - (1000 per page)\n", + "Processing 827 of 1000 on page 1 - (1000 per page)\n", + "Processing 828 of 1000 on page 1 - (1000 per page)\n", + "Processing 829 of 1000 on page 1 - (1000 per page)\n", + "Processing 830 of 1000 on page 1 - (1000 per page)\n", + "Processing 831 of 1000 on page 1 - (1000 per page)\n", + "Processing 832 of 1000 on page 1 - (1000 per page)\n", + "Processing 833 of 1000 on page 1 - (1000 per page)\n", + "Processing 834 of 1000 on page 1 - (1000 per page)\n", + "Processing 835 of 1000 on page 1 - (1000 per page)\n", + "Processing 836 of 1000 on page 1 - (1000 per page)\n", + "Processing 837 of 1000 on page 1 - (1000 per page)\n", + "Processing 838 of 1000 on page 1 - (1000 per page)\n", + "Processing 839 of 1000 on page 1 - (1000 per page)\n", + "Processing 840 of 1000 on page 1 - (1000 per page)\n", + "Processing 841 of 1000 on page 1 - (1000 per page)\n", + "Processing 842 of 1000 on page 1 - (1000 per page)\n", + "Processing 843 of 1000 on page 1 - (1000 per page)\n", + "Processing 844 of 1000 on page 1 - (1000 per page)\n", + "Processing 845 of 1000 on page 1 - (1000 per page)\n", + "Processing 846 of 1000 on page 1 - (1000 per page)\n", + "Processing 847 of 1000 on page 1 - (1000 per page)\n", + "Processing 848 of 1000 on page 1 - (1000 per page)\n", + "Processing 849 of 1000 on page 1 - (1000 per page)\n", + "Processing 850 of 1000 on page 1 - (1000 per page)\n", + "Processing 851 of 1000 on page 1 - (1000 per page)\n", + "Processing 852 of 1000 on page 1 - (1000 per page)\n", + "Processing 853 of 1000 on page 1 - (1000 per page)\n", + "Processing 854 of 1000 on page 1 - (1000 per page)\n", + "Processing 855 of 1000 on page 1 - (1000 per page)\n", + "Processing 856 of 1000 on page 1 - (1000 per page)\n", + "Processing 857 of 1000 on page 1 - (1000 per page)\n", + "Processing 858 of 1000 on page 1 - (1000 per page)\n", + "Processing 859 of 1000 on page 1 - (1000 per page)\n", + "Processing 860 of 1000 on page 1 - (1000 per page)\n", + "Processing 861 of 1000 on page 1 - (1000 per page)\n", + "Processing 862 of 1000 on page 1 - (1000 per page)\n", + "Processing 863 of 1000 on page 1 - (1000 per page)\n", + "Processing 864 of 1000 on page 1 - (1000 per page)\n", + "Processing 865 of 1000 on page 1 - (1000 per page)\n", + "Processing 866 of 1000 on page 1 - (1000 per page)\n", + "Processing 867 of 1000 on page 1 - (1000 per page)\n", + "Processing 868 of 1000 on page 1 - (1000 per page)\n", + "Processing 869 of 1000 on page 1 - (1000 per page)\n", + "Processing 870 of 1000 on page 1 - (1000 per page)\n", + "Processing 871 of 1000 on page 1 - (1000 per page)\n", + "Processing 872 of 1000 on page 1 - (1000 per page)\n", + "Processing 873 of 1000 on page 1 - (1000 per page)\n", + "Processing 874 of 1000 on page 1 - (1000 per page)\n", + "Processing 875 of 1000 on page 1 - (1000 per page)\n", + "Processing 876 of 1000 on page 1 - (1000 per page)\n", + "Processing 877 of 1000 on page 1 - (1000 per page)\n", + "Processing 878 of 1000 on page 1 - (1000 per page)\n", + "Processing 879 of 1000 on page 1 - (1000 per page)\n", + "Processing 880 of 1000 on page 1 - (1000 per page)\n", + "Processing 881 of 1000 on page 1 - (1000 per page)\n", + "Processing 882 of 1000 on page 1 - (1000 per page)\n", + "Processing 883 of 1000 on page 1 - (1000 per page)\n", + "Processing 884 of 1000 on page 1 - (1000 per page)\n", + "Processing 885 of 1000 on page 1 - (1000 per page)\n", + "Processing 886 of 1000 on page 1 - (1000 per page)\n", + "Processing 887 of 1000 on page 1 - (1000 per page)\n", + "Processing 888 of 1000 on page 1 - (1000 per page)\n", + "Processing 889 of 1000 on page 1 - (1000 per page)\n", + "Processing 890 of 1000 on page 1 - (1000 per page)\n", + "Processing 891 of 1000 on page 1 - (1000 per page)\n", + "Processing 892 of 1000 on page 1 - (1000 per page)\n", + "Processing 893 of 1000 on page 1 - (1000 per page)\n", + "Processing 894 of 1000 on page 1 - (1000 per page)\n", + "Processing 895 of 1000 on page 1 - (1000 per page)\n", + "Processing 896 of 1000 on page 1 - (1000 per page)\n", + "Processing 897 of 1000 on page 1 - (1000 per page)\n", + "Processing 898 of 1000 on page 1 - (1000 per page)\n", + "Processing 899 of 1000 on page 1 - (1000 per page)\n", + "Processing 900 of 1000 on page 1 - (1000 per page)\n", + "Processing 901 of 1000 on page 1 - (1000 per page)\n", + "Processing 902 of 1000 on page 1 - (1000 per page)\n", + "Processing 903 of 1000 on page 1 - (1000 per page)\n", + "Processing 904 of 1000 on page 1 - (1000 per page)\n", + "Processing 905 of 1000 on page 1 - (1000 per page)\n", + "Processing 906 of 1000 on page 1 - (1000 per page)\n", + "Processing 907 of 1000 on page 1 - (1000 per page)\n", + "Processing 908 of 1000 on page 1 - (1000 per page)\n", + "Processing 909 of 1000 on page 1 - (1000 per page)\n", + "Processing 910 of 1000 on page 1 - (1000 per page)\n", + "Processing 911 of 1000 on page 1 - (1000 per page)\n", + "Processing 912 of 1000 on page 1 - (1000 per page)\n", + "Processing 913 of 1000 on page 1 - (1000 per page)\n", + "Processing 914 of 1000 on page 1 - (1000 per page)\n", + "Processing 915 of 1000 on page 1 - (1000 per page)\n", + "Processing 916 of 1000 on page 1 - (1000 per page)\n", + "Processing 917 of 1000 on page 1 - (1000 per page)\n", + "Processing 918 of 1000 on page 1 - (1000 per page)\n", + "Processing 919 of 1000 on page 1 - (1000 per page)\n", + "Processing 920 of 1000 on page 1 - (1000 per page)\n", + "Processing 921 of 1000 on page 1 - (1000 per page)\n", + "Processing 922 of 1000 on page 1 - (1000 per page)\n", + "Processing 923 of 1000 on page 1 - (1000 per page)\n", + "Processing 924 of 1000 on page 1 - (1000 per page)\n", + "Processing 925 of 1000 on page 1 - (1000 per page)\n", + "Processing 926 of 1000 on page 1 - (1000 per page)\n", + "Processing 927 of 1000 on page 1 - (1000 per page)\n", + "Processing 928 of 1000 on page 1 - (1000 per page)\n", + "Processing 929 of 1000 on page 1 - (1000 per page)\n", + "Processing 930 of 1000 on page 1 - (1000 per page)\n", + "Processing 931 of 1000 on page 1 - (1000 per page)\n", + "Processing 932 of 1000 on page 1 - (1000 per page)\n", + "Processing 933 of 1000 on page 1 - (1000 per page)\n", + "Processing 934 of 1000 on page 1 - (1000 per page)\n", + "Processing 935 of 1000 on page 1 - (1000 per page)\n", + "Processing 936 of 1000 on page 1 - (1000 per page)\n", + "Processing 937 of 1000 on page 1 - (1000 per page)\n", + "Processing 938 of 1000 on page 1 - (1000 per page)\n", + "Processing 939 of 1000 on page 1 - (1000 per page)\n", + "Processing 940 of 1000 on page 1 - (1000 per page)\n", + "Processing 941 of 1000 on page 1 - (1000 per page)\n", + "Processing 942 of 1000 on page 1 - (1000 per page)\n", + "Processing 943 of 1000 on page 1 - (1000 per page)\n", + "Processing 944 of 1000 on page 1 - (1000 per page)\n", + "Processing 945 of 1000 on page 1 - (1000 per page)\n", + "Processing 946 of 1000 on page 1 - (1000 per page)\n", + "Processing 947 of 1000 on page 1 - (1000 per page)\n", + "Processing 948 of 1000 on page 1 - (1000 per page)\n", + "Processing 949 of 1000 on page 1 - (1000 per page)\n", + "Processing 950 of 1000 on page 1 - (1000 per page)\n", + "Processing 951 of 1000 on page 1 - (1000 per page)\n", + "Processing 952 of 1000 on page 1 - (1000 per page)\n", + "Processing 953 of 1000 on page 1 - (1000 per page)\n", + "Processing 954 of 1000 on page 1 - (1000 per page)\n", + "Processing 955 of 1000 on page 1 - (1000 per page)\n", + "Processing 956 of 1000 on page 1 - (1000 per page)\n", + "Processing 957 of 1000 on page 1 - (1000 per page)\n", + "Processing 958 of 1000 on page 1 - (1000 per page)\n", + "Processing 959 of 1000 on page 1 - (1000 per page)\n", + "Processing 960 of 1000 on page 1 - (1000 per page)\n", + "Processing 961 of 1000 on page 1 - (1000 per page)\n", + "Processing 962 of 1000 on page 1 - (1000 per page)\n", + "Processing 963 of 1000 on page 1 - (1000 per page)\n", + "Processing 964 of 1000 on page 1 - (1000 per page)\n", + "Processing 965 of 1000 on page 1 - (1000 per page)\n", + "Processing 966 of 1000 on page 1 - (1000 per page)\n", + "Processing 967 of 1000 on page 1 - (1000 per page)\n", + "Processing 968 of 1000 on page 1 - (1000 per page)\n", + "Processing 969 of 1000 on page 1 - (1000 per page)\n", + "Processing 970 of 1000 on page 1 - (1000 per page)\n", + "Processing 971 of 1000 on page 1 - (1000 per page)\n", + "Processing 972 of 1000 on page 1 - (1000 per page)\n", + "Processing 973 of 1000 on page 1 - (1000 per page)\n", + "Processing 974 of 1000 on page 1 - (1000 per page)\n", + "Processing 975 of 1000 on page 1 - (1000 per page)\n", + "Processing 976 of 1000 on page 1 - (1000 per page)\n", + "Processing 977 of 1000 on page 1 - (1000 per page)\n", + "Processing 978 of 1000 on page 1 - (1000 per page)\n", + "Processing 979 of 1000 on page 1 - (1000 per page)\n", + "Processing 980 of 1000 on page 1 - (1000 per page)\n", + "Processing 981 of 1000 on page 1 - (1000 per page)\n", + "Processing 982 of 1000 on page 1 - (1000 per page)\n", + "Processing 983 of 1000 on page 1 - (1000 per page)\n", + "Processing 984 of 1000 on page 1 - (1000 per page)\n", + "Processing 985 of 1000 on page 1 - (1000 per page)\n", + "Processing 986 of 1000 on page 1 - (1000 per page)\n", + "Processing 987 of 1000 on page 1 - (1000 per page)\n", + "Processing 988 of 1000 on page 1 - (1000 per page)\n", + "Processing 989 of 1000 on page 1 - (1000 per page)\n", + "Processing 990 of 1000 on page 1 - (1000 per page)\n", + "Processing 991 of 1000 on page 1 - (1000 per page)\n", + "Processing 992 of 1000 on page 1 - (1000 per page)\n", + "Processing 993 of 1000 on page 1 - (1000 per page)\n", + "Processing 994 of 1000 on page 1 - (1000 per page)\n", + "Processing 995 of 1000 on page 1 - (1000 per page)\n", + "Processing 996 of 1000 on page 1 - (1000 per page)\n", + "Processing 997 of 1000 on page 1 - (1000 per page)\n", + "Processing 998 of 1000 on page 1 - (1000 per page)\n", + "Processing 999 of 1000 on page 1 - (1000 per page)\n", + "Processing 1000 of 1000 on page 1 - (1000 per page)\n", + "Processing 1 of 1000 on page 2 - (1000 per page)\n", + "Processing 2 of 1000 on page 2 - (1000 per page)\n", + "Processing 3 of 1000 on page 2 - (1000 per page)\n", + "Processing 4 of 1000 on page 2 - (1000 per page)\n", + "Processing 5 of 1000 on page 2 - (1000 per page)\n", + "Processing 6 of 1000 on page 2 - (1000 per page)\n", + "Processing 7 of 1000 on page 2 - (1000 per page)\n", + "Processing 8 of 1000 on page 2 - (1000 per page)\n", + "Processing 9 of 1000 on page 2 - (1000 per page)\n", + "Processing 10 of 1000 on page 2 - (1000 per page)\n", + "Processing 11 of 1000 on page 2 - (1000 per page)\n", + "Processing 12 of 1000 on page 2 - (1000 per page)\n", + "Processing 13 of 1000 on page 2 - (1000 per page)\n", + "Processing 14 of 1000 on page 2 - (1000 per page)\n", + "Processing 15 of 1000 on page 2 - (1000 per page)\n", + "Processing 16 of 1000 on page 2 - (1000 per page)\n", + "Processing 17 of 1000 on page 2 - (1000 per page)\n", + "Processing 18 of 1000 on page 2 - (1000 per page)\n", + "Processing 19 of 1000 on page 2 - (1000 per page)\n", + "Processing 20 of 1000 on page 2 - (1000 per page)\n", + "Processing 21 of 1000 on page 2 - (1000 per page)\n", + "Processing 22 of 1000 on page 2 - (1000 per page)\n", + "Processing 23 of 1000 on page 2 - (1000 per page)\n", + "Processing 24 of 1000 on page 2 - (1000 per page)\n", + "Processing 25 of 1000 on page 2 - (1000 per page)\n", + "Processing 26 of 1000 on page 2 - (1000 per page)\n", + "Processing 27 of 1000 on page 2 - (1000 per page)\n", + "Processing 28 of 1000 on page 2 - (1000 per page)\n", + "Processing 29 of 1000 on page 2 - (1000 per page)\n", + "Processing 30 of 1000 on page 2 - (1000 per page)\n", + "Processing 31 of 1000 on page 2 - (1000 per page)\n", + "Processing 32 of 1000 on page 2 - (1000 per page)\n", + "Processing 33 of 1000 on page 2 - (1000 per page)\n", + "Processing 34 of 1000 on page 2 - (1000 per page)\n", + "Processing 35 of 1000 on page 2 - (1000 per page)\n", + "Processing 36 of 1000 on page 2 - (1000 per page)\n", + "Processing 37 of 1000 on page 2 - (1000 per page)\n", + "Processing 38 of 1000 on page 2 - (1000 per page)\n", + "Processing 39 of 1000 on page 2 - (1000 per page)\n", + "Processing 40 of 1000 on page 2 - (1000 per page)\n", + "Processing 41 of 1000 on page 2 - (1000 per page)\n", + "Processing 42 of 1000 on page 2 - (1000 per page)\n", + "Processing 43 of 1000 on page 2 - (1000 per page)\n", + "Processing 44 of 1000 on page 2 - (1000 per page)\n", + "Processing 45 of 1000 on page 2 - (1000 per page)\n", + "Processing 46 of 1000 on page 2 - (1000 per page)\n", + "Processing 47 of 1000 on page 2 - (1000 per page)\n", + "Processing 48 of 1000 on page 2 - (1000 per page)\n", + "Processing 49 of 1000 on page 2 - (1000 per page)\n", + "Processing 50 of 1000 on page 2 - (1000 per page)\n", + "Processing 51 of 1000 on page 2 - (1000 per page)\n", + "Processing 52 of 1000 on page 2 - (1000 per page)\n", + "Processing 53 of 1000 on page 2 - (1000 per page)\n", + "Processing 54 of 1000 on page 2 - (1000 per page)\n", + "Processing 55 of 1000 on page 2 - (1000 per page)\n", + "Processing 56 of 1000 on page 2 - (1000 per page)\n", + "Processing 57 of 1000 on page 2 - (1000 per page)\n", + "Processing 58 of 1000 on page 2 - (1000 per page)\n", + "Processing 59 of 1000 on page 2 - (1000 per page)\n", + "Processing 60 of 1000 on page 2 - (1000 per page)\n", + "Processing 61 of 1000 on page 2 - (1000 per page)\n", + "Processing 62 of 1000 on page 2 - (1000 per page)\n", + "Processing 63 of 1000 on page 2 - (1000 per page)\n", + "Processing 64 of 1000 on page 2 - (1000 per page)\n", + "Processing 65 of 1000 on page 2 - (1000 per page)\n", + "Processing 66 of 1000 on page 2 - (1000 per page)\n", + "Processing 67 of 1000 on page 2 - (1000 per page)\n", + "Processing 68 of 1000 on page 2 - (1000 per page)\n", + "Processing 69 of 1000 on page 2 - (1000 per page)\n", + "Processing 70 of 1000 on page 2 - (1000 per page)\n", + "Processing 71 of 1000 on page 2 - (1000 per page)\n", + "Processing 72 of 1000 on page 2 - (1000 per page)\n", + "Processing 73 of 1000 on page 2 - (1000 per page)\n", + "Processing 74 of 1000 on page 2 - (1000 per page)\n", + "Processing 75 of 1000 on page 2 - (1000 per page)\n", + "Processing 76 of 1000 on page 2 - (1000 per page)\n", + "Processing 77 of 1000 on page 2 - (1000 per page)\n", + "Processing 78 of 1000 on page 2 - (1000 per page)\n", + "Processing 79 of 1000 on page 2 - (1000 per page)\n", + "Processing 80 of 1000 on page 2 - (1000 per page)\n", + "Processing 81 of 1000 on page 2 - (1000 per page)\n", + "Processing 82 of 1000 on page 2 - (1000 per page)\n", + "Processing 83 of 1000 on page 2 - (1000 per page)\n", + "Processing 84 of 1000 on page 2 - (1000 per page)\n", + "Processing 85 of 1000 on page 2 - (1000 per page)\n", + "Processing 86 of 1000 on page 2 - (1000 per page)\n", + "Processing 87 of 1000 on page 2 - (1000 per page)\n", + "Processing 88 of 1000 on page 2 - (1000 per page)\n", + "Processing 89 of 1000 on page 2 - (1000 per page)\n", + "Processing 90 of 1000 on page 2 - (1000 per page)\n", + "Processing 91 of 1000 on page 2 - (1000 per page)\n", + "Processing 92 of 1000 on page 2 - (1000 per page)\n", + "Processing 93 of 1000 on page 2 - (1000 per page)\n", + "Processing 94 of 1000 on page 2 - (1000 per page)\n", + "Processing 95 of 1000 on page 2 - (1000 per page)\n", + "Processing 96 of 1000 on page 2 - (1000 per page)\n", + "Processing 97 of 1000 on page 2 - (1000 per page)\n", + "Processing 98 of 1000 on page 2 - (1000 per page)\n", + "Processing 99 of 1000 on page 2 - (1000 per page)\n", + "Processing 100 of 1000 on page 2 - (1000 per page)\n", + "Processing 101 of 1000 on page 2 - (1000 per page)\n", + "Processing 102 of 1000 on page 2 - (1000 per page)\n", + "Processing 103 of 1000 on page 2 - (1000 per page)\n", + "Processing 104 of 1000 on page 2 - (1000 per page)\n", + "Processing 105 of 1000 on page 2 - (1000 per page)\n", + "Processing 106 of 1000 on page 2 - (1000 per page)\n", + "Processing 107 of 1000 on page 2 - (1000 per page)\n", + "Processing 108 of 1000 on page 2 - (1000 per page)\n", + "Processing 109 of 1000 on page 2 - (1000 per page)\n", + "Processing 110 of 1000 on page 2 - (1000 per page)\n", + "Processing 111 of 1000 on page 2 - (1000 per page)\n", + "Processing 112 of 1000 on page 2 - (1000 per page)\n", + "Processing 113 of 1000 on page 2 - (1000 per page)\n", + "Processing 114 of 1000 on page 2 - (1000 per page)\n", + "Processing 115 of 1000 on page 2 - (1000 per page)\n", + "Processing 116 of 1000 on page 2 - (1000 per page)\n", + "Processing 117 of 1000 on page 2 - (1000 per page)\n", + "Processing 118 of 1000 on page 2 - (1000 per page)\n", + "Processing 119 of 1000 on page 2 - (1000 per page)\n", + "Processing 120 of 1000 on page 2 - (1000 per page)\n", + "Processing 121 of 1000 on page 2 - (1000 per page)\n", + "Processing 122 of 1000 on page 2 - (1000 per page)\n", + "Processing 123 of 1000 on page 2 - (1000 per page)\n", + "Processing 124 of 1000 on page 2 - (1000 per page)\n", + "Processing 125 of 1000 on page 2 - (1000 per page)\n", + "Processing 126 of 1000 on page 2 - (1000 per page)\n", + "Processing 127 of 1000 on page 2 - (1000 per page)\n", + "Processing 128 of 1000 on page 2 - (1000 per page)\n", + "Processing 129 of 1000 on page 2 - (1000 per page)\n", + "Processing 130 of 1000 on page 2 - (1000 per page)\n", + "Processing 131 of 1000 on page 2 - (1000 per page)\n", + "Processing 132 of 1000 on page 2 - (1000 per page)\n", + "Processing 133 of 1000 on page 2 - (1000 per page)\n", + "Processing 134 of 1000 on page 2 - (1000 per page)\n", + "Processing 135 of 1000 on page 2 - (1000 per page)\n", + "Processing 136 of 1000 on page 2 - (1000 per page)\n", + "Processing 137 of 1000 on page 2 - (1000 per page)\n", + "Processing 138 of 1000 on page 2 - (1000 per page)\n", + "Processing 139 of 1000 on page 2 - (1000 per page)\n", + "Processing 140 of 1000 on page 2 - (1000 per page)\n", + "Processing 141 of 1000 on page 2 - (1000 per page)\n", + "Processing 142 of 1000 on page 2 - (1000 per page)\n", + "Processing 143 of 1000 on page 2 - (1000 per page)\n", + "Processing 144 of 1000 on page 2 - (1000 per page)\n", + "Processing 145 of 1000 on page 2 - (1000 per page)\n", + "Processing 146 of 1000 on page 2 - (1000 per page)\n", + "Processing 147 of 1000 on page 2 - (1000 per page)\n", + "Processing 148 of 1000 on page 2 - (1000 per page)\n", + "Processing 149 of 1000 on page 2 - (1000 per page)\n", + "Processing 150 of 1000 on page 2 - (1000 per page)\n", + "Processing 151 of 1000 on page 2 - (1000 per page)\n", + "Processing 152 of 1000 on page 2 - (1000 per page)\n", + "Processing 153 of 1000 on page 2 - (1000 per page)\n", + "Processing 154 of 1000 on page 2 - (1000 per page)\n", + "Processing 155 of 1000 on page 2 - (1000 per page)\n", + "Processing 156 of 1000 on page 2 - (1000 per page)\n", + "Processing 157 of 1000 on page 2 - (1000 per page)\n", + "Processing 158 of 1000 on page 2 - (1000 per page)\n", + "Processing 159 of 1000 on page 2 - (1000 per page)\n", + "Processing 160 of 1000 on page 2 - (1000 per page)\n", + "Processing 161 of 1000 on page 2 - (1000 per page)\n", + "Processing 162 of 1000 on page 2 - (1000 per page)\n", + "Processing 163 of 1000 on page 2 - (1000 per page)\n", + "Processing 164 of 1000 on page 2 - (1000 per page)\n", + "Processing 165 of 1000 on page 2 - (1000 per page)\n", + "Processing 166 of 1000 on page 2 - (1000 per page)\n", + "Processing 167 of 1000 on page 2 - (1000 per page)\n", + "Processing 168 of 1000 on page 2 - (1000 per page)\n", + "Processing 169 of 1000 on page 2 - (1000 per page)\n", + "Processing 170 of 1000 on page 2 - (1000 per page)\n", + "Processing 171 of 1000 on page 2 - (1000 per page)\n", + "Processing 172 of 1000 on page 2 - (1000 per page)\n", + "Processing 173 of 1000 on page 2 - (1000 per page)\n", + "Processing 174 of 1000 on page 2 - (1000 per page)\n", + "Processing 175 of 1000 on page 2 - (1000 per page)\n", + "Processing 176 of 1000 on page 2 - (1000 per page)\n", + "Processing 177 of 1000 on page 2 - (1000 per page)\n", + "Processing 178 of 1000 on page 2 - (1000 per page)\n", + "Processing 179 of 1000 on page 2 - (1000 per page)\n", + "Processing 180 of 1000 on page 2 - (1000 per page)\n", + "Processing 181 of 1000 on page 2 - (1000 per page)\n", + "Processing 182 of 1000 on page 2 - (1000 per page)\n", + "Processing 183 of 1000 on page 2 - (1000 per page)\n", + "Processing 184 of 1000 on page 2 - (1000 per page)\n", + "Processing 185 of 1000 on page 2 - (1000 per page)\n", + "Processing 186 of 1000 on page 2 - (1000 per page)\n", + "Processing 187 of 1000 on page 2 - (1000 per page)\n", + "Processing 188 of 1000 on page 2 - (1000 per page)\n", + "Processing 189 of 1000 on page 2 - (1000 per page)\n", + "Processing 190 of 1000 on page 2 - (1000 per page)\n", + "Processing 191 of 1000 on page 2 - (1000 per page)\n", + "Processing 192 of 1000 on page 2 - (1000 per page)\n", + "Processing 193 of 1000 on page 2 - (1000 per page)\n", + "Processing 194 of 1000 on page 2 - (1000 per page)\n", + "Processing 195 of 1000 on page 2 - (1000 per page)\n", + "Processing 196 of 1000 on page 2 - (1000 per page)\n", + "Processing 197 of 1000 on page 2 - (1000 per page)\n", + "Processing 198 of 1000 on page 2 - (1000 per page)\n", + "Processing 199 of 1000 on page 2 - (1000 per page)\n", + "Processing 200 of 1000 on page 2 - (1000 per page)\n", + "Processing 201 of 1000 on page 2 - (1000 per page)\n", + "Processing 202 of 1000 on page 2 - (1000 per page)\n", + "Processing 203 of 1000 on page 2 - (1000 per page)\n", + "Processing 204 of 1000 on page 2 - (1000 per page)\n", + "Processing 205 of 1000 on page 2 - (1000 per page)\n", + "Processing 206 of 1000 on page 2 - (1000 per page)\n", + "Processing 207 of 1000 on page 2 - (1000 per page)\n", + "Processing 208 of 1000 on page 2 - (1000 per page)\n", + "Processing 209 of 1000 on page 2 - (1000 per page)\n", + "Processing 210 of 1000 on page 2 - (1000 per page)\n", + "Processing 211 of 1000 on page 2 - (1000 per page)\n", + "Processing 212 of 1000 on page 2 - (1000 per page)\n", + "Processing 213 of 1000 on page 2 - (1000 per page)\n", + "Processing 214 of 1000 on page 2 - (1000 per page)\n", + "Processing 215 of 1000 on page 2 - (1000 per page)\n", + "Processing 216 of 1000 on page 2 - (1000 per page)\n", + "Processing 217 of 1000 on page 2 - (1000 per page)\n", + "Processing 218 of 1000 on page 2 - (1000 per page)\n", + "Processing 219 of 1000 on page 2 - (1000 per page)\n", + "Processing 220 of 1000 on page 2 - (1000 per page)\n", + "Processing 221 of 1000 on page 2 - (1000 per page)\n", + "Processing 222 of 1000 on page 2 - (1000 per page)\n", + "Processing 223 of 1000 on page 2 - (1000 per page)\n", + "Processing 224 of 1000 on page 2 - (1000 per page)\n", + "Processing 225 of 1000 on page 2 - (1000 per page)\n", + "Processing 226 of 1000 on page 2 - (1000 per page)\n", + "Processing 227 of 1000 on page 2 - (1000 per page)\n", + "Processing 228 of 1000 on page 2 - (1000 per page)\n", + "Processing 229 of 1000 on page 2 - (1000 per page)\n", + "Processing 230 of 1000 on page 2 - (1000 per page)\n", + "Processing 231 of 1000 on page 2 - (1000 per page)\n", + "Processing 232 of 1000 on page 2 - (1000 per page)\n", + "Processing 233 of 1000 on page 2 - (1000 per page)\n", + "Processing 234 of 1000 on page 2 - (1000 per page)\n", + "Processing 235 of 1000 on page 2 - (1000 per page)\n", + "Processing 236 of 1000 on page 2 - (1000 per page)\n", + "Processing 237 of 1000 on page 2 - (1000 per page)\n", + "Processing 238 of 1000 on page 2 - (1000 per page)\n", + "Processing 239 of 1000 on page 2 - (1000 per page)\n", + "Processing 240 of 1000 on page 2 - (1000 per page)\n", + "Processing 241 of 1000 on page 2 - (1000 per page)\n", + "Processing 242 of 1000 on page 2 - (1000 per page)\n", + "Processing 243 of 1000 on page 2 - (1000 per page)\n", + "Processing 244 of 1000 on page 2 - (1000 per page)\n", + "Processing 245 of 1000 on page 2 - (1000 per page)\n", + "Processing 246 of 1000 on page 2 - (1000 per page)\n", + "Processing 247 of 1000 on page 2 - (1000 per page)\n", + "Processing 248 of 1000 on page 2 - (1000 per page)\n", + "Processing 249 of 1000 on page 2 - (1000 per page)\n", + "Processing 250 of 1000 on page 2 - (1000 per page)\n", + "Processing 251 of 1000 on page 2 - (1000 per page)\n", + "Processing 252 of 1000 on page 2 - (1000 per page)\n", + "Processing 253 of 1000 on page 2 - (1000 per page)\n", + "Processing 254 of 1000 on page 2 - (1000 per page)\n", + "Processing 255 of 1000 on page 2 - (1000 per page)\n", + "Processing 256 of 1000 on page 2 - (1000 per page)\n", + "Processing 257 of 1000 on page 2 - (1000 per page)\n", + "Processing 258 of 1000 on page 2 - (1000 per page)\n", + "Processing 259 of 1000 on page 2 - (1000 per page)\n", + "Processing 260 of 1000 on page 2 - (1000 per page)\n", + "Processing 261 of 1000 on page 2 - (1000 per page)\n", + "Processing 262 of 1000 on page 2 - (1000 per page)\n", + "Processing 263 of 1000 on page 2 - (1000 per page)\n", + "Processing 264 of 1000 on page 2 - (1000 per page)\n", + "Processing 265 of 1000 on page 2 - (1000 per page)\n", + "Processing 266 of 1000 on page 2 - (1000 per page)\n", + "Processing 267 of 1000 on page 2 - (1000 per page)\n", + "Processing 268 of 1000 on page 2 - (1000 per page)\n", + "Processing 269 of 1000 on page 2 - (1000 per page)\n", + "Processing 270 of 1000 on page 2 - (1000 per page)\n", + "Processing 271 of 1000 on page 2 - (1000 per page)\n", + "Processing 272 of 1000 on page 2 - (1000 per page)\n", + "Processing 273 of 1000 on page 2 - (1000 per page)\n", + "Processing 274 of 1000 on page 2 - (1000 per page)\n", + "Processing 275 of 1000 on page 2 - (1000 per page)\n", + "Processing 276 of 1000 on page 2 - (1000 per page)\n", + "Processing 277 of 1000 on page 2 - (1000 per page)\n", + "Processing 278 of 1000 on page 2 - (1000 per page)\n", + "Processing 279 of 1000 on page 2 - (1000 per page)\n", + "Processing 280 of 1000 on page 2 - (1000 per page)\n", + "Processing 281 of 1000 on page 2 - (1000 per page)\n", + "Processing 282 of 1000 on page 2 - (1000 per page)\n", + "Processing 283 of 1000 on page 2 - (1000 per page)\n", + "Processing 284 of 1000 on page 2 - (1000 per page)\n", + "Processing 285 of 1000 on page 2 - (1000 per page)\n", + "Processing 286 of 1000 on page 2 - (1000 per page)\n", + "Processing 287 of 1000 on page 2 - (1000 per page)\n", + "Processing 288 of 1000 on page 2 - (1000 per page)\n", + "Processing 289 of 1000 on page 2 - (1000 per page)\n", + "Processing 290 of 1000 on page 2 - (1000 per page)\n", + "Processing 291 of 1000 on page 2 - (1000 per page)\n", + "Processing 292 of 1000 on page 2 - (1000 per page)\n", + "Processing 293 of 1000 on page 2 - (1000 per page)\n", + "Processing 294 of 1000 on page 2 - (1000 per page)\n", + "Processing 295 of 1000 on page 2 - (1000 per page)\n", + "Processing 296 of 1000 on page 2 - (1000 per page)\n", + "Processing 297 of 1000 on page 2 - (1000 per page)\n", + "Processing 298 of 1000 on page 2 - (1000 per page)\n", + "Processing 299 of 1000 on page 2 - (1000 per page)\n", + "Processing 300 of 1000 on page 2 - (1000 per page)\n", + "Processing 301 of 1000 on page 2 - (1000 per page)\n", + "Processing 302 of 1000 on page 2 - (1000 per page)\n", + "Processing 303 of 1000 on page 2 - (1000 per page)\n", + "Processing 304 of 1000 on page 2 - (1000 per page)\n", + "Processing 305 of 1000 on page 2 - (1000 per page)\n", + "Processing 306 of 1000 on page 2 - (1000 per page)\n", + "Processing 307 of 1000 on page 2 - (1000 per page)\n", + "Processing 308 of 1000 on page 2 - (1000 per page)\n", + "Processing 309 of 1000 on page 2 - (1000 per page)\n", + "Processing 310 of 1000 on page 2 - (1000 per page)\n", + "Processing 311 of 1000 on page 2 - (1000 per page)\n", + "Processing 312 of 1000 on page 2 - (1000 per page)\n", + "Processing 313 of 1000 on page 2 - (1000 per page)\n", + "Processing 314 of 1000 on page 2 - (1000 per page)\n", + "Processing 315 of 1000 on page 2 - (1000 per page)\n", + "Processing 316 of 1000 on page 2 - (1000 per page)\n", + "Processing 317 of 1000 on page 2 - (1000 per page)\n", + "Processing 318 of 1000 on page 2 - (1000 per page)\n", + "Processing 319 of 1000 on page 2 - (1000 per page)\n", + "Processing 320 of 1000 on page 2 - (1000 per page)\n", + "Processing 321 of 1000 on page 2 - (1000 per page)\n", + "Processing 322 of 1000 on page 2 - (1000 per page)\n", + "Processing 323 of 1000 on page 2 - (1000 per page)\n", + "Processing 324 of 1000 on page 2 - (1000 per page)\n", + "Processing 325 of 1000 on page 2 - (1000 per page)\n", + "Processing 326 of 1000 on page 2 - (1000 per page)\n", + "Processing 327 of 1000 on page 2 - (1000 per page)\n", + "Processing 328 of 1000 on page 2 - (1000 per page)\n", + "Processing 329 of 1000 on page 2 - (1000 per page)\n", + "Processing 330 of 1000 on page 2 - (1000 per page)\n", + "Processing 331 of 1000 on page 2 - (1000 per page)\n", + "Processing 332 of 1000 on page 2 - (1000 per page)\n", + "Processing 333 of 1000 on page 2 - (1000 per page)\n", + "Processing 334 of 1000 on page 2 - (1000 per page)\n", + "Processing 335 of 1000 on page 2 - (1000 per page)\n", + "Processing 336 of 1000 on page 2 - (1000 per page)\n", + "Processing 337 of 1000 on page 2 - (1000 per page)\n", + "Processing 338 of 1000 on page 2 - (1000 per page)\n", + "Processing 339 of 1000 on page 2 - (1000 per page)\n", + "Processing 340 of 1000 on page 2 - (1000 per page)\n", + "Processing 341 of 1000 on page 2 - (1000 per page)\n", + "Processing 342 of 1000 on page 2 - (1000 per page)\n", + "Processing 343 of 1000 on page 2 - (1000 per page)\n", + "Processing 344 of 1000 on page 2 - (1000 per page)\n", + "Processing 345 of 1000 on page 2 - (1000 per page)\n", + "Processing 346 of 1000 on page 2 - (1000 per page)\n", + "Processing 347 of 1000 on page 2 - (1000 per page)\n", + "Processing 348 of 1000 on page 2 - (1000 per page)\n", + "Processing 349 of 1000 on page 2 - (1000 per page)\n", + "Processing 350 of 1000 on page 2 - (1000 per page)\n", + "Processing 351 of 1000 on page 2 - (1000 per page)\n", + "Processing 352 of 1000 on page 2 - (1000 per page)\n", + "Processing 353 of 1000 on page 2 - (1000 per page)\n", + "Processing 354 of 1000 on page 2 - (1000 per page)\n", + "Processing 355 of 1000 on page 2 - (1000 per page)\n", + "Processing 356 of 1000 on page 2 - (1000 per page)\n", + "Processing 357 of 1000 on page 2 - (1000 per page)\n", + "Processing 358 of 1000 on page 2 - (1000 per page)\n", + "Processing 359 of 1000 on page 2 - (1000 per page)\n", + "Processing 360 of 1000 on page 2 - (1000 per page)\n", + "Processing 361 of 1000 on page 2 - (1000 per page)\n", + "Processing 362 of 1000 on page 2 - (1000 per page)\n", + "Processing 363 of 1000 on page 2 - (1000 per page)\n", + "Processing 364 of 1000 on page 2 - (1000 per page)\n", + "Processing 365 of 1000 on page 2 - (1000 per page)\n", + "Processing 366 of 1000 on page 2 - (1000 per page)\n", + "Processing 367 of 1000 on page 2 - (1000 per page)\n", + "Processing 368 of 1000 on page 2 - (1000 per page)\n", + "Processing 369 of 1000 on page 2 - (1000 per page)\n", + "Processing 370 of 1000 on page 2 - (1000 per page)\n", + "Processing 371 of 1000 on page 2 - (1000 per page)\n", + "Processing 372 of 1000 on page 2 - (1000 per page)\n", + "Processing 373 of 1000 on page 2 - (1000 per page)\n", + "Processing 374 of 1000 on page 2 - (1000 per page)\n", + "Processing 375 of 1000 on page 2 - (1000 per page)\n", + "Processing 376 of 1000 on page 2 - (1000 per page)\n", + "Processing 377 of 1000 on page 2 - (1000 per page)\n", + "Processing 378 of 1000 on page 2 - (1000 per page)\n", + "Processing 379 of 1000 on page 2 - (1000 per page)\n", + "Processing 380 of 1000 on page 2 - (1000 per page)\n", + "Processing 381 of 1000 on page 2 - (1000 per page)\n", + "Processing 382 of 1000 on page 2 - (1000 per page)\n", + "Processing 383 of 1000 on page 2 - (1000 per page)\n", + "Processing 384 of 1000 on page 2 - (1000 per page)\n", + "Processing 385 of 1000 on page 2 - (1000 per page)\n", + "Processing 386 of 1000 on page 2 - (1000 per page)\n", + "Processing 387 of 1000 on page 2 - (1000 per page)\n", + "Processing 388 of 1000 on page 2 - (1000 per page)\n", + "Processing 389 of 1000 on page 2 - (1000 per page)\n", + "Processing 390 of 1000 on page 2 - (1000 per page)\n", + "Processing 391 of 1000 on page 2 - (1000 per page)\n", + "Processing 392 of 1000 on page 2 - (1000 per page)\n", + "Processing 393 of 1000 on page 2 - (1000 per page)\n", + "Processing 394 of 1000 on page 2 - (1000 per page)\n", + "Processing 395 of 1000 on page 2 - (1000 per page)\n", + "Processing 396 of 1000 on page 2 - (1000 per page)\n", + "Processing 397 of 1000 on page 2 - (1000 per page)\n", + "Processing 398 of 1000 on page 2 - (1000 per page)\n", + "Processing 399 of 1000 on page 2 - (1000 per page)\n", + "Processing 400 of 1000 on page 2 - (1000 per page)\n", + "Processing 401 of 1000 on page 2 - (1000 per page)\n", + "Processing 402 of 1000 on page 2 - (1000 per page)\n", + "Processing 403 of 1000 on page 2 - (1000 per page)\n", + "Processing 404 of 1000 on page 2 - (1000 per page)\n", + "Processing 405 of 1000 on page 2 - (1000 per page)\n", + "Processing 406 of 1000 on page 2 - (1000 per page)\n", + "Processing 407 of 1000 on page 2 - (1000 per page)\n", + "Processing 408 of 1000 on page 2 - (1000 per page)\n", + "Processing 409 of 1000 on page 2 - (1000 per page)\n", + "Processing 410 of 1000 on page 2 - (1000 per page)\n", + "Processing 411 of 1000 on page 2 - (1000 per page)\n", + "Processing 412 of 1000 on page 2 - (1000 per page)\n", + "Processing 413 of 1000 on page 2 - (1000 per page)\n", + "Processing 414 of 1000 on page 2 - (1000 per page)\n", + "Processing 415 of 1000 on page 2 - (1000 per page)\n", + "Processing 416 of 1000 on page 2 - (1000 per page)\n", + "Processing 417 of 1000 on page 2 - (1000 per page)\n", + "Processing 418 of 1000 on page 2 - (1000 per page)\n", + "Processing 419 of 1000 on page 2 - (1000 per page)\n", + "Processing 420 of 1000 on page 2 - (1000 per page)\n", + "Processing 421 of 1000 on page 2 - (1000 per page)\n", + "Processing 422 of 1000 on page 2 - (1000 per page)\n", + "Processing 423 of 1000 on page 2 - (1000 per page)\n", + "Processing 424 of 1000 on page 2 - (1000 per page)\n", + "Processing 425 of 1000 on page 2 - (1000 per page)\n", + "Processing 426 of 1000 on page 2 - (1000 per page)\n", + "Processing 427 of 1000 on page 2 - (1000 per page)\n", + "Processing 428 of 1000 on page 2 - (1000 per page)\n", + "Processing 429 of 1000 on page 2 - (1000 per page)\n", + "Processing 430 of 1000 on page 2 - (1000 per page)\n", + "Processing 431 of 1000 on page 2 - (1000 per page)\n", + "Processing 432 of 1000 on page 2 - (1000 per page)\n", + "Processing 433 of 1000 on page 2 - (1000 per page)\n", + "Processing 434 of 1000 on page 2 - (1000 per page)\n", + "Processing 435 of 1000 on page 2 - (1000 per page)\n", + "Processing 436 of 1000 on page 2 - (1000 per page)\n", + "Processing 437 of 1000 on page 2 - (1000 per page)\n", + "Processing 438 of 1000 on page 2 - (1000 per page)\n", + "Processing 439 of 1000 on page 2 - (1000 per page)\n", + "Processing 440 of 1000 on page 2 - (1000 per page)\n", + "Processing 441 of 1000 on page 2 - (1000 per page)\n", + "Processing 442 of 1000 on page 2 - (1000 per page)\n", + "Processing 443 of 1000 on page 2 - (1000 per page)\n", + "Processing 444 of 1000 on page 2 - (1000 per page)\n", + "Processing 445 of 1000 on page 2 - (1000 per page)\n", + "Processing 446 of 1000 on page 2 - (1000 per page)\n", + "Processing 447 of 1000 on page 2 - (1000 per page)\n", + "Processing 448 of 1000 on page 2 - (1000 per page)\n", + "Processing 449 of 1000 on page 2 - (1000 per page)\n", + "Processing 450 of 1000 on page 2 - (1000 per page)\n", + "Processing 451 of 1000 on page 2 - (1000 per page)\n", + "Processing 452 of 1000 on page 2 - (1000 per page)\n", + "Processing 453 of 1000 on page 2 - (1000 per page)\n", + "Processing 454 of 1000 on page 2 - (1000 per page)\n", + "Processing 455 of 1000 on page 2 - (1000 per page)\n", + "Processing 456 of 1000 on page 2 - (1000 per page)\n", + "Processing 457 of 1000 on page 2 - (1000 per page)\n", + "Processing 458 of 1000 on page 2 - (1000 per page)\n", + "Processing 459 of 1000 on page 2 - (1000 per page)\n", + "Processing 460 of 1000 on page 2 - (1000 per page)\n", + "Processing 461 of 1000 on page 2 - (1000 per page)\n", + "Processing 462 of 1000 on page 2 - (1000 per page)\n", + "Processing 463 of 1000 on page 2 - (1000 per page)\n", + "Processing 464 of 1000 on page 2 - (1000 per page)\n", + "Processing 465 of 1000 on page 2 - (1000 per page)\n", + "Processing 466 of 1000 on page 2 - (1000 per page)\n", + "Processing 467 of 1000 on page 2 - (1000 per page)\n", + "Processing 468 of 1000 on page 2 - (1000 per page)\n", + "Processing 469 of 1000 on page 2 - (1000 per page)\n", + "Processing 470 of 1000 on page 2 - (1000 per page)\n", + "Processing 471 of 1000 on page 2 - (1000 per page)\n", + "Processing 472 of 1000 on page 2 - (1000 per page)\n", + "Processing 473 of 1000 on page 2 - (1000 per page)\n", + "Processing 474 of 1000 on page 2 - (1000 per page)\n", + "Processing 475 of 1000 on page 2 - (1000 per page)\n", + "Processing 476 of 1000 on page 2 - (1000 per page)\n", + "Processing 477 of 1000 on page 2 - (1000 per page)\n", + "Processing 478 of 1000 on page 2 - (1000 per page)\n", + "Processing 479 of 1000 on page 2 - (1000 per page)\n", + "Processing 480 of 1000 on page 2 - (1000 per page)\n", + "Processing 481 of 1000 on page 2 - (1000 per page)\n", + "Processing 482 of 1000 on page 2 - (1000 per page)\n", + "Processing 483 of 1000 on page 2 - (1000 per page)\n", + "Processing 484 of 1000 on page 2 - (1000 per page)\n", + "Processing 485 of 1000 on page 2 - (1000 per page)\n", + "Processing 486 of 1000 on page 2 - (1000 per page)\n", + "Processing 487 of 1000 on page 2 - (1000 per page)\n", + "Processing 488 of 1000 on page 2 - (1000 per page)\n", + "Processing 489 of 1000 on page 2 - (1000 per page)\n", + "Processing 490 of 1000 on page 2 - (1000 per page)\n", + "Processing 491 of 1000 on page 2 - (1000 per page)\n", + "Processing 492 of 1000 on page 2 - (1000 per page)\n", + "Processing 493 of 1000 on page 2 - (1000 per page)\n", + "Processing 494 of 1000 on page 2 - (1000 per page)\n", + "Processing 495 of 1000 on page 2 - (1000 per page)\n", + "Processing 496 of 1000 on page 2 - (1000 per page)\n", + "Processing 497 of 1000 on page 2 - (1000 per page)\n", + "Processing 498 of 1000 on page 2 - (1000 per page)\n", + "Processing 499 of 1000 on page 2 - (1000 per page)\n", + "Processing 500 of 1000 on page 2 - (1000 per page)\n", + "Processing 501 of 1000 on page 2 - (1000 per page)\n", + "Processing 502 of 1000 on page 2 - (1000 per page)\n", + "Processing 503 of 1000 on page 2 - (1000 per page)\n", + "Processing 504 of 1000 on page 2 - (1000 per page)\n", + "Processing 505 of 1000 on page 2 - (1000 per page)\n", + "Processing 506 of 1000 on page 2 - (1000 per page)\n", + "Processing 507 of 1000 on page 2 - (1000 per page)\n", + "Processing 508 of 1000 on page 2 - (1000 per page)\n", + "Processing 509 of 1000 on page 2 - (1000 per page)\n", + "Processing 510 of 1000 on page 2 - (1000 per page)\n", + "Processing 511 of 1000 on page 2 - (1000 per page)\n", + "Processing 512 of 1000 on page 2 - (1000 per page)\n", + "Processing 513 of 1000 on page 2 - (1000 per page)\n", + "Processing 514 of 1000 on page 2 - (1000 per page)\n", + "Processing 515 of 1000 on page 2 - (1000 per page)\n", + "Processing 516 of 1000 on page 2 - (1000 per page)\n", + "Processing 517 of 1000 on page 2 - (1000 per page)\n", + "Processing 518 of 1000 on page 2 - (1000 per page)\n", + "Processing 519 of 1000 on page 2 - (1000 per page)\n", + "Processing 520 of 1000 on page 2 - (1000 per page)\n", + "Processing 521 of 1000 on page 2 - (1000 per page)\n", + "Processing 522 of 1000 on page 2 - (1000 per page)\n", + "Processing 523 of 1000 on page 2 - (1000 per page)\n", + "Processing 524 of 1000 on page 2 - (1000 per page)\n", + "Processing 525 of 1000 on page 2 - (1000 per page)\n", + "Processing 526 of 1000 on page 2 - (1000 per page)\n", + "Processing 527 of 1000 on page 2 - (1000 per page)\n", + "Processing 528 of 1000 on page 2 - (1000 per page)\n", + "Processing 529 of 1000 on page 2 - (1000 per page)\n", + "Processing 530 of 1000 on page 2 - (1000 per page)\n", + "Processing 531 of 1000 on page 2 - (1000 per page)\n", + "Processing 532 of 1000 on page 2 - (1000 per page)\n", + "Processing 533 of 1000 on page 2 - (1000 per page)\n", + "Processing 534 of 1000 on page 2 - (1000 per page)\n", + "Processing 535 of 1000 on page 2 - (1000 per page)\n", + "Processing 536 of 1000 on page 2 - (1000 per page)\n", + "Processing 537 of 1000 on page 2 - (1000 per page)\n", + "Processing 538 of 1000 on page 2 - (1000 per page)\n", + "Processing 539 of 1000 on page 2 - (1000 per page)\n", + "Processing 540 of 1000 on page 2 - (1000 per page)\n", + "Processing 541 of 1000 on page 2 - (1000 per page)\n", + "Processing 542 of 1000 on page 2 - (1000 per page)\n", + "Processing 543 of 1000 on page 2 - (1000 per page)\n", + "Processing 544 of 1000 on page 2 - (1000 per page)\n", + "Processing 545 of 1000 on page 2 - (1000 per page)\n", + "Processing 546 of 1000 on page 2 - (1000 per page)\n", + "Processing 547 of 1000 on page 2 - (1000 per page)\n", + "Processing 548 of 1000 on page 2 - (1000 per page)\n", + "Processing 549 of 1000 on page 2 - (1000 per page)\n", + "Processing 550 of 1000 on page 2 - (1000 per page)\n", + "Processing 551 of 1000 on page 2 - (1000 per page)\n", + "Processing 552 of 1000 on page 2 - (1000 per page)\n", + "Processing 553 of 1000 on page 2 - (1000 per page)\n", + "Processing 554 of 1000 on page 2 - (1000 per page)\n", + "Processing 555 of 1000 on page 2 - (1000 per page)\n", + "Processing 556 of 1000 on page 2 - (1000 per page)\n", + "Processing 557 of 1000 on page 2 - (1000 per page)\n", + "Processing 558 of 1000 on page 2 - (1000 per page)\n", + "Processing 559 of 1000 on page 2 - (1000 per page)\n", + "Processing 560 of 1000 on page 2 - (1000 per page)\n", + "Processing 561 of 1000 on page 2 - (1000 per page)\n", + "Processing 562 of 1000 on page 2 - (1000 per page)\n", + "Processing 563 of 1000 on page 2 - (1000 per page)\n", + "Processing 564 of 1000 on page 2 - (1000 per page)\n", + "Processing 565 of 1000 on page 2 - (1000 per page)\n", + "Processing 566 of 1000 on page 2 - (1000 per page)\n", + "Processing 567 of 1000 on page 2 - (1000 per page)\n", + "Processing 568 of 1000 on page 2 - (1000 per page)\n", + "Processing 569 of 1000 on page 2 - (1000 per page)\n", + "Processing 570 of 1000 on page 2 - (1000 per page)\n", + "Processing 571 of 1000 on page 2 - (1000 per page)\n", + "Processing 572 of 1000 on page 2 - (1000 per page)\n", + "Processing 573 of 1000 on page 2 - (1000 per page)\n", + "Processing 574 of 1000 on page 2 - (1000 per page)\n", + "Processing 575 of 1000 on page 2 - (1000 per page)\n", + "Processing 576 of 1000 on page 2 - (1000 per page)\n", + "Processing 577 of 1000 on page 2 - (1000 per page)\n", + "Processing 578 of 1000 on page 2 - (1000 per page)\n", + "Processing 579 of 1000 on page 2 - (1000 per page)\n", + "Processing 580 of 1000 on page 2 - (1000 per page)\n", + "Processing 581 of 1000 on page 2 - (1000 per page)\n", + "Processing 582 of 1000 on page 2 - (1000 per page)\n", + "Processing 583 of 1000 on page 2 - (1000 per page)\n", + "Processing 584 of 1000 on page 2 - (1000 per page)\n", + "Processing 585 of 1000 on page 2 - (1000 per page)\n", + "Processing 586 of 1000 on page 2 - (1000 per page)\n", + "Processing 587 of 1000 on page 2 - (1000 per page)\n", + "Processing 588 of 1000 on page 2 - (1000 per page)\n", + "Processing 589 of 1000 on page 2 - (1000 per page)\n", + "Processing 590 of 1000 on page 2 - (1000 per page)\n", + "Processing 591 of 1000 on page 2 - (1000 per page)\n", + "Processing 592 of 1000 on page 2 - (1000 per page)\n", + "Processing 593 of 1000 on page 2 - (1000 per page)\n", + "Processing 594 of 1000 on page 2 - (1000 per page)\n", + "Processing 595 of 1000 on page 2 - (1000 per page)\n", + "Processing 596 of 1000 on page 2 - (1000 per page)\n", + "Processing 597 of 1000 on page 2 - (1000 per page)\n", + "Processing 598 of 1000 on page 2 - (1000 per page)\n", + "Processing 599 of 1000 on page 2 - (1000 per page)\n", + "Processing 600 of 1000 on page 2 - (1000 per page)\n", + "Processing 601 of 1000 on page 2 - (1000 per page)\n", + "Processing 602 of 1000 on page 2 - (1000 per page)\n", + "Processing 603 of 1000 on page 2 - (1000 per page)\n", + "Processing 604 of 1000 on page 2 - (1000 per page)\n", + "Processing 605 of 1000 on page 2 - (1000 per page)\n", + "Processing 606 of 1000 on page 2 - (1000 per page)\n", + "Processing 607 of 1000 on page 2 - (1000 per page)\n", + "Processing 608 of 1000 on page 2 - (1000 per page)\n", + "Processing 609 of 1000 on page 2 - (1000 per page)\n", + "Processing 610 of 1000 on page 2 - (1000 per page)\n", + "Processing 611 of 1000 on page 2 - (1000 per page)\n", + "Processing 612 of 1000 on page 2 - (1000 per page)\n", + "Processing 613 of 1000 on page 2 - (1000 per page)\n", + "Processing 614 of 1000 on page 2 - (1000 per page)\n", + "Processing 615 of 1000 on page 2 - (1000 per page)\n", + "Processing 616 of 1000 on page 2 - (1000 per page)\n", + "Processing 617 of 1000 on page 2 - (1000 per page)\n", + "Processing 618 of 1000 on page 2 - (1000 per page)\n", + "Processing 619 of 1000 on page 2 - (1000 per page)\n", + "Processing 620 of 1000 on page 2 - (1000 per page)\n", + "Processing 621 of 1000 on page 2 - (1000 per page)\n", + "Processing 622 of 1000 on page 2 - (1000 per page)\n", + "Processing 623 of 1000 on page 2 - (1000 per page)\n", + "Processing 624 of 1000 on page 2 - (1000 per page)\n", + "Processing 625 of 1000 on page 2 - (1000 per page)\n", + "Processing 626 of 1000 on page 2 - (1000 per page)\n", + "Processing 627 of 1000 on page 2 - (1000 per page)\n", + "Processing 628 of 1000 on page 2 - (1000 per page)\n", + "Processing 629 of 1000 on page 2 - (1000 per page)\n", + "Processing 630 of 1000 on page 2 - (1000 per page)\n", + "Processing 631 of 1000 on page 2 - (1000 per page)\n", + "Processing 632 of 1000 on page 2 - (1000 per page)\n", + "Processing 633 of 1000 on page 2 - (1000 per page)\n", + "Processing 634 of 1000 on page 2 - (1000 per page)\n", + "Processing 635 of 1000 on page 2 - (1000 per page)\n", + "Processing 636 of 1000 on page 2 - (1000 per page)\n", + "Processing 637 of 1000 on page 2 - (1000 per page)\n", + "Processing 638 of 1000 on page 2 - (1000 per page)\n", + "Processing 639 of 1000 on page 2 - (1000 per page)\n", + "Processing 640 of 1000 on page 2 - (1000 per page)\n", + "Processing 641 of 1000 on page 2 - (1000 per page)\n", + "Processing 642 of 1000 on page 2 - (1000 per page)\n", + "Processing 643 of 1000 on page 2 - (1000 per page)\n", + "Processing 644 of 1000 on page 2 - (1000 per page)\n", + "Processing 645 of 1000 on page 2 - (1000 per page)\n", + "Processing 646 of 1000 on page 2 - (1000 per page)\n", + "Processing 647 of 1000 on page 2 - (1000 per page)\n", + "Processing 648 of 1000 on page 2 - (1000 per page)\n", + "Processing 649 of 1000 on page 2 - (1000 per page)\n", + "Processing 650 of 1000 on page 2 - (1000 per page)\n", + "Processing 651 of 1000 on page 2 - (1000 per page)\n", + "Processing 652 of 1000 on page 2 - (1000 per page)\n", + "Processing 653 of 1000 on page 2 - (1000 per page)\n", + "Processing 654 of 1000 on page 2 - (1000 per page)\n", + "Processing 655 of 1000 on page 2 - (1000 per page)\n", + "Processing 656 of 1000 on page 2 - (1000 per page)\n", + "Processing 657 of 1000 on page 2 - (1000 per page)\n", + "Processing 658 of 1000 on page 2 - (1000 per page)\n", + "Processing 659 of 1000 on page 2 - (1000 per page)\n", + "Processing 660 of 1000 on page 2 - (1000 per page)\n", + "Processing 661 of 1000 on page 2 - (1000 per page)\n", + "Processing 662 of 1000 on page 2 - (1000 per page)\n", + "Processing 663 of 1000 on page 2 - (1000 per page)\n", + "Processing 664 of 1000 on page 2 - (1000 per page)\n", + "Processing 665 of 1000 on page 2 - (1000 per page)\n", + "Processing 666 of 1000 on page 2 - (1000 per page)\n", + "Processing 667 of 1000 on page 2 - (1000 per page)\n", + "Processing 668 of 1000 on page 2 - (1000 per page)\n", + "Processing 669 of 1000 on page 2 - (1000 per page)\n", + "Processing 670 of 1000 on page 2 - (1000 per page)\n", + "Processing 671 of 1000 on page 2 - (1000 per page)\n", + "Processing 672 of 1000 on page 2 - (1000 per page)\n", + "Processing 673 of 1000 on page 2 - (1000 per page)\n", + "Processing 674 of 1000 on page 2 - (1000 per page)\n", + "Processing 675 of 1000 on page 2 - (1000 per page)\n", + "Processing 676 of 1000 on page 2 - (1000 per page)\n", + "Processing 677 of 1000 on page 2 - (1000 per page)\n", + "Processing 678 of 1000 on page 2 - (1000 per page)\n", + "Processing 679 of 1000 on page 2 - (1000 per page)\n", + "Processing 680 of 1000 on page 2 - (1000 per page)\n", + "Processing 681 of 1000 on page 2 - (1000 per page)\n", + "Processing 682 of 1000 on page 2 - (1000 per page)\n", + "Processing 683 of 1000 on page 2 - (1000 per page)\n", + "Processing 684 of 1000 on page 2 - (1000 per page)\n", + "Processing 685 of 1000 on page 2 - (1000 per page)\n", + "Processing 686 of 1000 on page 2 - (1000 per page)\n", + "Processing 687 of 1000 on page 2 - (1000 per page)\n", + "Processing 688 of 1000 on page 2 - (1000 per page)\n", + "Processing 689 of 1000 on page 2 - (1000 per page)\n", + "Processing 690 of 1000 on page 2 - (1000 per page)\n", + "Processing 691 of 1000 on page 2 - (1000 per page)\n", + "Processing 692 of 1000 on page 2 - (1000 per page)\n", + "Processing 693 of 1000 on page 2 - (1000 per page)\n", + "Processing 694 of 1000 on page 2 - (1000 per page)\n", + "Processing 695 of 1000 on page 2 - (1000 per page)\n", + "Processing 696 of 1000 on page 2 - (1000 per page)\n", + "Processing 697 of 1000 on page 2 - (1000 per page)\n", + "Processing 698 of 1000 on page 2 - (1000 per page)\n", + "Processing 699 of 1000 on page 2 - (1000 per page)\n", + "Processing 700 of 1000 on page 2 - (1000 per page)\n", + "Processing 701 of 1000 on page 2 - (1000 per page)\n", + "Processing 702 of 1000 on page 2 - (1000 per page)\n", + "Processing 703 of 1000 on page 2 - (1000 per page)\n", + "Processing 704 of 1000 on page 2 - (1000 per page)\n", + "Processing 705 of 1000 on page 2 - (1000 per page)\n", + "Processing 706 of 1000 on page 2 - (1000 per page)\n", + "Processing 707 of 1000 on page 2 - (1000 per page)\n", + "Processing 708 of 1000 on page 2 - (1000 per page)\n", + "Processing 709 of 1000 on page 2 - (1000 per page)\n", + "Processing 710 of 1000 on page 2 - (1000 per page)\n", + "Processing 711 of 1000 on page 2 - (1000 per page)\n", + "Processing 712 of 1000 on page 2 - (1000 per page)\n", + "Processing 713 of 1000 on page 2 - (1000 per page)\n", + "Processing 714 of 1000 on page 2 - (1000 per page)\n", + "Processing 715 of 1000 on page 2 - (1000 per page)\n", + "Processing 716 of 1000 on page 2 - (1000 per page)\n", + "Processing 717 of 1000 on page 2 - (1000 per page)\n", + "Processing 718 of 1000 on page 2 - (1000 per page)\n", + "Processing 719 of 1000 on page 2 - (1000 per page)\n", + "Processing 720 of 1000 on page 2 - (1000 per page)\n", + "Processing 721 of 1000 on page 2 - (1000 per page)\n", + "Processing 722 of 1000 on page 2 - (1000 per page)\n", + "Processing 723 of 1000 on page 2 - (1000 per page)\n", + "Processing 724 of 1000 on page 2 - (1000 per page)\n", + "Processing 725 of 1000 on page 2 - (1000 per page)\n", + "Processing 726 of 1000 on page 2 - (1000 per page)\n", + "Processing 727 of 1000 on page 2 - (1000 per page)\n", + "Processing 728 of 1000 on page 2 - (1000 per page)\n", + "Processing 729 of 1000 on page 2 - (1000 per page)\n", + "Processing 730 of 1000 on page 2 - (1000 per page)\n", + "Processing 731 of 1000 on page 2 - (1000 per page)\n", + "Processing 732 of 1000 on page 2 - (1000 per page)\n", + "Processing 733 of 1000 on page 2 - (1000 per page)\n", + "Processing 734 of 1000 on page 2 - (1000 per page)\n", + "Processing 735 of 1000 on page 2 - (1000 per page)\n", + "Processing 736 of 1000 on page 2 - (1000 per page)\n", + "Processing 737 of 1000 on page 2 - (1000 per page)\n", + "Processing 738 of 1000 on page 2 - (1000 per page)\n", + "Processing 739 of 1000 on page 2 - (1000 per page)\n", + "Processing 740 of 1000 on page 2 - (1000 per page)\n", + "Processing 741 of 1000 on page 2 - (1000 per page)\n", + "Processing 742 of 1000 on page 2 - (1000 per page)\n", + "Processing 743 of 1000 on page 2 - (1000 per page)\n", + "Processing 744 of 1000 on page 2 - (1000 per page)\n", + "Processing 745 of 1000 on page 2 - (1000 per page)\n", + "Processing 746 of 1000 on page 2 - (1000 per page)\n", + "Processing 747 of 1000 on page 2 - (1000 per page)\n", + "Processing 748 of 1000 on page 2 - (1000 per page)\n", + "Processing 749 of 1000 on page 2 - (1000 per page)\n", + "Processing 750 of 1000 on page 2 - (1000 per page)\n", + "Processing 751 of 1000 on page 2 - (1000 per page)\n", + "Processing 752 of 1000 on page 2 - (1000 per page)\n", + "Processing 753 of 1000 on page 2 - (1000 per page)\n", + "Processing 754 of 1000 on page 2 - (1000 per page)\n", + "Processing 755 of 1000 on page 2 - (1000 per page)\n", + "Processing 756 of 1000 on page 2 - (1000 per page)\n", + "Processing 757 of 1000 on page 2 - (1000 per page)\n", + "Processing 758 of 1000 on page 2 - (1000 per page)\n", + "Processing 759 of 1000 on page 2 - (1000 per page)\n", + "Processing 760 of 1000 on page 2 - (1000 per page)\n", + "Processing 761 of 1000 on page 2 - (1000 per page)\n", + "Processing 762 of 1000 on page 2 - (1000 per page)\n", + "Processing 763 of 1000 on page 2 - (1000 per page)\n", + "Processing 764 of 1000 on page 2 - (1000 per page)\n", + "Processing 765 of 1000 on page 2 - (1000 per page)\n", + "Processing 766 of 1000 on page 2 - (1000 per page)\n", + "Processing 767 of 1000 on page 2 - (1000 per page)\n", + "Processing 768 of 1000 on page 2 - (1000 per page)\n", + "Processing 769 of 1000 on page 2 - (1000 per page)\n", + "Processing 770 of 1000 on page 2 - (1000 per page)\n", + "Processing 771 of 1000 on page 2 - (1000 per page)\n", + "Processing 772 of 1000 on page 2 - (1000 per page)\n", + "Processing 773 of 1000 on page 2 - (1000 per page)\n", + "Processing 774 of 1000 on page 2 - (1000 per page)\n", + "Processing 775 of 1000 on page 2 - (1000 per page)\n", + "Processing 776 of 1000 on page 2 - (1000 per page)\n", + "Processing 777 of 1000 on page 2 - (1000 per page)\n", + "Processing 778 of 1000 on page 2 - (1000 per page)\n", + "Processing 779 of 1000 on page 2 - (1000 per page)\n", + "Processing 780 of 1000 on page 2 - (1000 per page)\n", + "Processing 781 of 1000 on page 2 - (1000 per page)\n", + "Processing 782 of 1000 on page 2 - (1000 per page)\n", + "Processing 783 of 1000 on page 2 - (1000 per page)\n", + "Processing 784 of 1000 on page 2 - (1000 per page)\n", + "Processing 785 of 1000 on page 2 - (1000 per page)\n", + "Processing 786 of 1000 on page 2 - (1000 per page)\n", + "Processing 787 of 1000 on page 2 - (1000 per page)\n", + "Processing 788 of 1000 on page 2 - (1000 per page)\n", + "Processing 789 of 1000 on page 2 - (1000 per page)\n", + "Processing 790 of 1000 on page 2 - (1000 per page)\n", + "Processing 791 of 1000 on page 2 - (1000 per page)\n", + "Processing 792 of 1000 on page 2 - (1000 per page)\n", + "Processing 793 of 1000 on page 2 - (1000 per page)\n", + "Processing 794 of 1000 on page 2 - (1000 per page)\n", + "Processing 795 of 1000 on page 2 - (1000 per page)\n", + "Processing 796 of 1000 on page 2 - (1000 per page)\n", + "Processing 797 of 1000 on page 2 - (1000 per page)\n", + "Processing 798 of 1000 on page 2 - (1000 per page)\n", + "Processing 799 of 1000 on page 2 - (1000 per page)\n", + "Processing 800 of 1000 on page 2 - (1000 per page)\n", + "Processing 801 of 1000 on page 2 - (1000 per page)\n", + "Processing 802 of 1000 on page 2 - (1000 per page)\n", + "Processing 803 of 1000 on page 2 - (1000 per page)\n", + "Processing 804 of 1000 on page 2 - (1000 per page)\n", + "Processing 805 of 1000 on page 2 - (1000 per page)\n", + "Processing 806 of 1000 on page 2 - (1000 per page)\n", + "Processing 807 of 1000 on page 2 - (1000 per page)\n", + "Processing 808 of 1000 on page 2 - (1000 per page)\n", + "Processing 809 of 1000 on page 2 - (1000 per page)\n", + "Processing 810 of 1000 on page 2 - (1000 per page)\n", + "Processing 811 of 1000 on page 2 - (1000 per page)\n", + "Processing 812 of 1000 on page 2 - (1000 per page)\n", + "Processing 813 of 1000 on page 2 - (1000 per page)\n", + "Processing 814 of 1000 on page 2 - (1000 per page)\n", + "Processing 815 of 1000 on page 2 - (1000 per page)\n", + "Processing 816 of 1000 on page 2 - (1000 per page)\n", + "Processing 817 of 1000 on page 2 - (1000 per page)\n", + "Processing 818 of 1000 on page 2 - (1000 per page)\n", + "Processing 819 of 1000 on page 2 - (1000 per page)\n", + "Processing 820 of 1000 on page 2 - (1000 per page)\n", + "Processing 821 of 1000 on page 2 - (1000 per page)\n", + "Processing 822 of 1000 on page 2 - (1000 per page)\n", + "Processing 823 of 1000 on page 2 - (1000 per page)\n", + "Processing 824 of 1000 on page 2 - (1000 per page)\n", + "Processing 825 of 1000 on page 2 - (1000 per page)\n", + "Processing 826 of 1000 on page 2 - (1000 per page)\n", + "Processing 827 of 1000 on page 2 - (1000 per page)\n", + "Processing 828 of 1000 on page 2 - (1000 per page)\n", + "Processing 829 of 1000 on page 2 - (1000 per page)\n", + "Processing 830 of 1000 on page 2 - (1000 per page)\n", + "Processing 831 of 1000 on page 2 - (1000 per page)\n", + "Processing 832 of 1000 on page 2 - (1000 per page)\n", + "Processing 833 of 1000 on page 2 - (1000 per page)\n", + "Processing 834 of 1000 on page 2 - (1000 per page)\n", + "Processing 835 of 1000 on page 2 - (1000 per page)\n", + "Processing 836 of 1000 on page 2 - (1000 per page)\n", + "Processing 837 of 1000 on page 2 - (1000 per page)\n", + "Processing 838 of 1000 on page 2 - (1000 per page)\n", + "Processing 839 of 1000 on page 2 - (1000 per page)\n", + "Processing 840 of 1000 on page 2 - (1000 per page)\n", + "Processing 841 of 1000 on page 2 - (1000 per page)\n", + "Processing 842 of 1000 on page 2 - (1000 per page)\n", + "Processing 843 of 1000 on page 2 - (1000 per page)\n", + "Processing 844 of 1000 on page 2 - (1000 per page)\n", + "Processing 845 of 1000 on page 2 - (1000 per page)\n", + "Processing 846 of 1000 on page 2 - (1000 per page)\n", + "Processing 847 of 1000 on page 2 - (1000 per page)\n", + "Processing 848 of 1000 on page 2 - (1000 per page)\n", + "Processing 849 of 1000 on page 2 - (1000 per page)\n", + "Processing 850 of 1000 on page 2 - (1000 per page)\n", + "Processing 851 of 1000 on page 2 - (1000 per page)\n", + "Processing 852 of 1000 on page 2 - (1000 per page)\n", + "Processing 853 of 1000 on page 2 - (1000 per page)\n", + "Processing 854 of 1000 on page 2 - (1000 per page)\n", + "Processing 855 of 1000 on page 2 - (1000 per page)\n", + "Processing 856 of 1000 on page 2 - (1000 per page)\n", + "Processing 857 of 1000 on page 2 - (1000 per page)\n", + "Processing 858 of 1000 on page 2 - (1000 per page)\n", + "Processing 859 of 1000 on page 2 - (1000 per page)\n", + "Processing 860 of 1000 on page 2 - (1000 per page)\n", + "Processing 861 of 1000 on page 2 - (1000 per page)\n", + "Processing 862 of 1000 on page 2 - (1000 per page)\n", + "Processing 863 of 1000 on page 2 - (1000 per page)\n", + "Processing 864 of 1000 on page 2 - (1000 per page)\n", + "Processing 865 of 1000 on page 2 - (1000 per page)\n", + "Processing 866 of 1000 on page 2 - (1000 per page)\n", + "Processing 867 of 1000 on page 2 - (1000 per page)\n", + "Processing 868 of 1000 on page 2 - (1000 per page)\n", + "Processing 869 of 1000 on page 2 - (1000 per page)\n", + "Processing 870 of 1000 on page 2 - (1000 per page)\n", + "Processing 871 of 1000 on page 2 - (1000 per page)\n", + "Processing 872 of 1000 on page 2 - (1000 per page)\n", + "Processing 873 of 1000 on page 2 - (1000 per page)\n", + "Processing 874 of 1000 on page 2 - (1000 per page)\n", + "Processing 875 of 1000 on page 2 - (1000 per page)\n", + "Processing 876 of 1000 on page 2 - (1000 per page)\n", + "Processing 877 of 1000 on page 2 - (1000 per page)\n", + "Processing 878 of 1000 on page 2 - (1000 per page)\n", + "Processing 879 of 1000 on page 2 - (1000 per page)\n", + "Processing 880 of 1000 on page 2 - (1000 per page)\n", + "Processing 881 of 1000 on page 2 - (1000 per page)\n", + "Processing 882 of 1000 on page 2 - (1000 per page)\n", + "Processing 883 of 1000 on page 2 - (1000 per page)\n", + "Processing 884 of 1000 on page 2 - (1000 per page)\n", + "Processing 885 of 1000 on page 2 - (1000 per page)\n", + "Processing 886 of 1000 on page 2 - (1000 per page)\n", + "Processing 887 of 1000 on page 2 - (1000 per page)\n", + "Processing 888 of 1000 on page 2 - (1000 per page)\n", + "Processing 889 of 1000 on page 2 - (1000 per page)\n", + "Processing 890 of 1000 on page 2 - (1000 per page)\n", + "Processing 891 of 1000 on page 2 - (1000 per page)\n", + "Processing 892 of 1000 on page 2 - (1000 per page)\n", + "Processing 893 of 1000 on page 2 - (1000 per page)\n", + "Processing 894 of 1000 on page 2 - (1000 per page)\n", + "Processing 895 of 1000 on page 2 - (1000 per page)\n", + "Processing 896 of 1000 on page 2 - (1000 per page)\n", + "Processing 897 of 1000 on page 2 - (1000 per page)\n", + "Processing 898 of 1000 on page 2 - (1000 per page)\n", + "Processing 899 of 1000 on page 2 - (1000 per page)\n", + "Processing 900 of 1000 on page 2 - (1000 per page)\n", + "Processing 901 of 1000 on page 2 - (1000 per page)\n", + "Processing 902 of 1000 on page 2 - (1000 per page)\n", + "Processing 903 of 1000 on page 2 - (1000 per page)\n", + "Processing 904 of 1000 on page 2 - (1000 per page)\n", + "Processing 905 of 1000 on page 2 - (1000 per page)\n", + "Processing 906 of 1000 on page 2 - (1000 per page)\n", + "Processing 907 of 1000 on page 2 - (1000 per page)\n", + "Processing 908 of 1000 on page 2 - (1000 per page)\n", + "Processing 909 of 1000 on page 2 - (1000 per page)\n", + "Processing 910 of 1000 on page 2 - (1000 per page)\n", + "Processing 911 of 1000 on page 2 - (1000 per page)\n", + "Processing 912 of 1000 on page 2 - (1000 per page)\n", + "Processing 913 of 1000 on page 2 - (1000 per page)\n", + "Processing 914 of 1000 on page 2 - (1000 per page)\n", + "Processing 915 of 1000 on page 2 - (1000 per page)\n", + "Processing 916 of 1000 on page 2 - (1000 per page)\n", + "Processing 917 of 1000 on page 2 - (1000 per page)\n", + "Processing 918 of 1000 on page 2 - (1000 per page)\n", + "Processing 919 of 1000 on page 2 - (1000 per page)\n", + "Processing 920 of 1000 on page 2 - (1000 per page)\n", + "Processing 921 of 1000 on page 2 - (1000 per page)\n", + "Processing 922 of 1000 on page 2 - (1000 per page)\n", + "Processing 923 of 1000 on page 2 - (1000 per page)\n", + "Processing 924 of 1000 on page 2 - (1000 per page)\n", + "Processing 925 of 1000 on page 2 - (1000 per page)\n", + "Processing 926 of 1000 on page 2 - (1000 per page)\n", + "Processing 927 of 1000 on page 2 - (1000 per page)\n", + "Processing 928 of 1000 on page 2 - (1000 per page)\n", + "Processing 929 of 1000 on page 2 - (1000 per page)\n", + "Processing 930 of 1000 on page 2 - (1000 per page)\n", + "Processing 931 of 1000 on page 2 - (1000 per page)\n", + "Processing 932 of 1000 on page 2 - (1000 per page)\n", + "Processing 933 of 1000 on page 2 - (1000 per page)\n", + "Processing 934 of 1000 on page 2 - (1000 per page)\n", + "Processing 935 of 1000 on page 2 - (1000 per page)\n", + "Processing 936 of 1000 on page 2 - (1000 per page)\n", + "Processing 937 of 1000 on page 2 - (1000 per page)\n", + "Processing 938 of 1000 on page 2 - (1000 per page)\n", + "Processing 939 of 1000 on page 2 - (1000 per page)\n", + "Processing 940 of 1000 on page 2 - (1000 per page)\n", + "Processing 941 of 1000 on page 2 - (1000 per page)\n", + "Processing 942 of 1000 on page 2 - (1000 per page)\n", + "Processing 943 of 1000 on page 2 - (1000 per page)\n", + "Processing 944 of 1000 on page 2 - (1000 per page)\n", + "Processing 945 of 1000 on page 2 - (1000 per page)\n", + "Processing 946 of 1000 on page 2 - (1000 per page)\n", + "Processing 947 of 1000 on page 2 - (1000 per page)\n", + "Processing 948 of 1000 on page 2 - (1000 per page)\n", + "Processing 949 of 1000 on page 2 - (1000 per page)\n", + "Processing 950 of 1000 on page 2 - (1000 per page)\n", + "Processing 951 of 1000 on page 2 - (1000 per page)\n", + "Processing 952 of 1000 on page 2 - (1000 per page)\n", + "Processing 953 of 1000 on page 2 - (1000 per page)\n", + "Processing 954 of 1000 on page 2 - (1000 per page)\n", + "Processing 955 of 1000 on page 2 - (1000 per page)\n", + "Processing 956 of 1000 on page 2 - (1000 per page)\n", + "Processing 957 of 1000 on page 2 - (1000 per page)\n", + "Processing 958 of 1000 on page 2 - (1000 per page)\n", + "Processing 959 of 1000 on page 2 - (1000 per page)\n", + "Processing 960 of 1000 on page 2 - (1000 per page)\n", + "Processing 961 of 1000 on page 2 - (1000 per page)\n", + "Processing 962 of 1000 on page 2 - (1000 per page)\n", + "Processing 963 of 1000 on page 2 - (1000 per page)\n", + "Processing 964 of 1000 on page 2 - (1000 per page)\n", + "Processing 965 of 1000 on page 2 - (1000 per page)\n", + "Processing 966 of 1000 on page 2 - (1000 per page)\n", + "Processing 967 of 1000 on page 2 - (1000 per page)\n", + "Processing 968 of 1000 on page 2 - (1000 per page)\n", + "Processing 969 of 1000 on page 2 - (1000 per page)\n", + "Processing 970 of 1000 on page 2 - (1000 per page)\n", + "Processing 971 of 1000 on page 2 - (1000 per page)\n", + "Processing 972 of 1000 on page 2 - (1000 per page)\n", + "Processing 973 of 1000 on page 2 - (1000 per page)\n", + "Processing 974 of 1000 on page 2 - (1000 per page)\n", + "Processing 975 of 1000 on page 2 - (1000 per page)\n", + "Processing 976 of 1000 on page 2 - (1000 per page)\n", + "Processing 977 of 1000 on page 2 - (1000 per page)\n", + "Processing 978 of 1000 on page 2 - (1000 per page)\n", + "Processing 979 of 1000 on page 2 - (1000 per page)\n", + "Processing 980 of 1000 on page 2 - (1000 per page)\n", + "Processing 981 of 1000 on page 2 - (1000 per page)\n", + "Processing 982 of 1000 on page 2 - (1000 per page)\n", + "Processing 983 of 1000 on page 2 - (1000 per page)\n", + "Processing 984 of 1000 on page 2 - (1000 per page)\n", + "Processing 985 of 1000 on page 2 - (1000 per page)\n", + "Processing 986 of 1000 on page 2 - (1000 per page)\n", + "Processing 987 of 1000 on page 2 - (1000 per page)\n", + "Processing 988 of 1000 on page 2 - (1000 per page)\n", + "Processing 989 of 1000 on page 2 - (1000 per page)\n", + "Processing 990 of 1000 on page 2 - (1000 per page)\n", + "Processing 991 of 1000 on page 2 - (1000 per page)\n", + "Processing 992 of 1000 on page 2 - (1000 per page)\n", + "Processing 993 of 1000 on page 2 - (1000 per page)\n", + "Processing 994 of 1000 on page 2 - (1000 per page)\n", + "Processing 995 of 1000 on page 2 - (1000 per page)\n", + "Processing 996 of 1000 on page 2 - (1000 per page)\n", + "Processing 997 of 1000 on page 2 - (1000 per page)\n", + "Processing 998 of 1000 on page 2 - (1000 per page)\n", + "Processing 999 of 1000 on page 2 - (1000 per page)\n", + "Processing 1000 of 1000 on page 2 - (1000 per page)\n", + "Processing 1 of 153 on page 3 - (1000 per page)\n", + "Processing 2 of 153 on page 3 - (1000 per page)\n", + "Processing 3 of 153 on page 3 - (1000 per page)\n", + "Processing 4 of 153 on page 3 - (1000 per page)\n", + "Processing 5 of 153 on page 3 - (1000 per page)\n", + "Processing 6 of 153 on page 3 - (1000 per page)\n", + "Processing 7 of 153 on page 3 - (1000 per page)\n", + "Processing 8 of 153 on page 3 - (1000 per page)\n", + "Processing 9 of 153 on page 3 - (1000 per page)\n", + "Processing 10 of 153 on page 3 - (1000 per page)\n", + "Processing 11 of 153 on page 3 - (1000 per page)\n", + "Processing 12 of 153 on page 3 - (1000 per page)\n", + "Processing 13 of 153 on page 3 - (1000 per page)\n", + "Processing 14 of 153 on page 3 - (1000 per page)\n", + "Processing 15 of 153 on page 3 - (1000 per page)\n", + "Processing 16 of 153 on page 3 - (1000 per page)\n", + "Processing 17 of 153 on page 3 - (1000 per page)\n", + "Processing 18 of 153 on page 3 - (1000 per page)\n", + "Processing 19 of 153 on page 3 - (1000 per page)\n", + "Processing 20 of 153 on page 3 - (1000 per page)\n", + "Processing 21 of 153 on page 3 - (1000 per page)\n", + "Processing 22 of 153 on page 3 - (1000 per page)\n", + "Processing 23 of 153 on page 3 - (1000 per page)\n", + "Processing 24 of 153 on page 3 - (1000 per page)\n", + "Processing 25 of 153 on page 3 - (1000 per page)\n", + "Processing 26 of 153 on page 3 - (1000 per page)\n", + "Processing 27 of 153 on page 3 - (1000 per page)\n", + "Processing 28 of 153 on page 3 - (1000 per page)\n", + "Processing 29 of 153 on page 3 - (1000 per page)\n", + "Processing 30 of 153 on page 3 - (1000 per page)\n", + "Processing 31 of 153 on page 3 - (1000 per page)\n", + "Processing 32 of 153 on page 3 - (1000 per page)\n", + "Processing 33 of 153 on page 3 - (1000 per page)\n", + "Processing 34 of 153 on page 3 - (1000 per page)\n", + "Processing 35 of 153 on page 3 - (1000 per page)\n", + "Processing 36 of 153 on page 3 - (1000 per page)\n", + "Processing 37 of 153 on page 3 - (1000 per page)\n", + "Processing 38 of 153 on page 3 - (1000 per page)\n", + "Processing 39 of 153 on page 3 - (1000 per page)\n", + "Processing 40 of 153 on page 3 - (1000 per page)\n", + "Processing 41 of 153 on page 3 - (1000 per page)\n", + "Processing 42 of 153 on page 3 - (1000 per page)\n", + "Processing 43 of 153 on page 3 - (1000 per page)\n", + "Processing 44 of 153 on page 3 - (1000 per page)\n", + "Processing 45 of 153 on page 3 - (1000 per page)\n", + "Processing 46 of 153 on page 3 - (1000 per page)\n", + "Processing 47 of 153 on page 3 - (1000 per page)\n", + "Processing 48 of 153 on page 3 - (1000 per page)\n", + "Processing 49 of 153 on page 3 - (1000 per page)\n", + "Processing 50 of 153 on page 3 - (1000 per page)\n", + "Processing 51 of 153 on page 3 - (1000 per page)\n", + "Processing 52 of 153 on page 3 - (1000 per page)\n", + "Processing 53 of 153 on page 3 - (1000 per page)\n", + "Processing 54 of 153 on page 3 - (1000 per page)\n", + "Processing 55 of 153 on page 3 - (1000 per page)\n", + "Processing 56 of 153 on page 3 - (1000 per page)\n", + "Processing 57 of 153 on page 3 - (1000 per page)\n", + "Processing 58 of 153 on page 3 - (1000 per page)\n", + "Processing 59 of 153 on page 3 - (1000 per page)\n", + "Processing 60 of 153 on page 3 - (1000 per page)\n", + "Processing 61 of 153 on page 3 - (1000 per page)\n", + "Processing 62 of 153 on page 3 - (1000 per page)\n", + "Processing 63 of 153 on page 3 - (1000 per page)\n", + "Processing 64 of 153 on page 3 - (1000 per page)\n", + "Processing 65 of 153 on page 3 - (1000 per page)\n", + "Processing 66 of 153 on page 3 - (1000 per page)\n", + "Processing 67 of 153 on page 3 - (1000 per page)\n", + "Processing 68 of 153 on page 3 - (1000 per page)\n", + "Processing 69 of 153 on page 3 - (1000 per page)\n", + "Processing 70 of 153 on page 3 - (1000 per page)\n", + "Processing 71 of 153 on page 3 - (1000 per page)\n", + "Processing 72 of 153 on page 3 - (1000 per page)\n", + "Processing 73 of 153 on page 3 - (1000 per page)\n", + "Processing 74 of 153 on page 3 - (1000 per page)\n", + "Processing 75 of 153 on page 3 - (1000 per page)\n", + "Processing 76 of 153 on page 3 - (1000 per page)\n", + "Processing 77 of 153 on page 3 - (1000 per page)\n", + "Processing 78 of 153 on page 3 - (1000 per page)\n", + "Processing 79 of 153 on page 3 - (1000 per page)\n", + "Processing 80 of 153 on page 3 - (1000 per page)\n", + "Processing 81 of 153 on page 3 - (1000 per page)\n", + "Processing 82 of 153 on page 3 - (1000 per page)\n", + "Processing 83 of 153 on page 3 - (1000 per page)\n", + "Processing 84 of 153 on page 3 - (1000 per page)\n", + "Processing 85 of 153 on page 3 - (1000 per page)\n", + "Processing 86 of 153 on page 3 - (1000 per page)\n", + "Processing 87 of 153 on page 3 - (1000 per page)\n", + "Processing 88 of 153 on page 3 - (1000 per page)\n", + "Processing 89 of 153 on page 3 - (1000 per page)\n", + "Processing 90 of 153 on page 3 - (1000 per page)\n", + "Processing 91 of 153 on page 3 - (1000 per page)\n", + "Processing 92 of 153 on page 3 - (1000 per page)\n", + "Processing 93 of 153 on page 3 - (1000 per page)\n", + "Processing 94 of 153 on page 3 - (1000 per page)\n", + "Processing 95 of 153 on page 3 - (1000 per page)\n", + "Processing 96 of 153 on page 3 - (1000 per page)\n", + "Processing 97 of 153 on page 3 - (1000 per page)\n", + "Processing 98 of 153 on page 3 - (1000 per page)\n", + "Processing 99 of 153 on page 3 - (1000 per page)\n", + "Processing 100 of 153 on page 3 - (1000 per page)\n", + "Processing 101 of 153 on page 3 - (1000 per page)\n", + "Processing 102 of 153 on page 3 - (1000 per page)\n", + "Processing 103 of 153 on page 3 - (1000 per page)\n", + "Processing 104 of 153 on page 3 - (1000 per page)\n", + "Processing 105 of 153 on page 3 - (1000 per page)\n", + "Processing 106 of 153 on page 3 - (1000 per page)\n", + "Processing 107 of 153 on page 3 - (1000 per page)\n", + "Processing 108 of 153 on page 3 - (1000 per page)\n", + "Processing 109 of 153 on page 3 - (1000 per page)\n", + "Processing 110 of 153 on page 3 - (1000 per page)\n", + "Processing 111 of 153 on page 3 - (1000 per page)\n", + "Processing 112 of 153 on page 3 - (1000 per page)\n", + "Processing 113 of 153 on page 3 - (1000 per page)\n", + "Processing 114 of 153 on page 3 - (1000 per page)\n", + "Processing 115 of 153 on page 3 - (1000 per page)\n", + "Processing 116 of 153 on page 3 - (1000 per page)\n", + "Processing 117 of 153 on page 3 - (1000 per page)\n", + "Processing 118 of 153 on page 3 - (1000 per page)\n", + "Processing 119 of 153 on page 3 - (1000 per page)\n", + "Processing 120 of 153 on page 3 - (1000 per page)\n", + "Processing 121 of 153 on page 3 - (1000 per page)\n", + "Processing 122 of 153 on page 3 - (1000 per page)\n", + "Processing 123 of 153 on page 3 - (1000 per page)\n", + "Processing 124 of 153 on page 3 - (1000 per page)\n", + "Processing 125 of 153 on page 3 - (1000 per page)\n", + "Processing 126 of 153 on page 3 - (1000 per page)\n", + "Processing 127 of 153 on page 3 - (1000 per page)\n", + "Processing 128 of 153 on page 3 - (1000 per page)\n", + "Processing 129 of 153 on page 3 - (1000 per page)\n", + "Processing 130 of 153 on page 3 - (1000 per page)\n", + "Processing 131 of 153 on page 3 - (1000 per page)\n", + "Processing 132 of 153 on page 3 - (1000 per page)\n", + "Processing 133 of 153 on page 3 - (1000 per page)\n", + "Processing 134 of 153 on page 3 - (1000 per page)\n", + "Processing 135 of 153 on page 3 - (1000 per page)\n", + "Processing 136 of 153 on page 3 - (1000 per page)\n", + "Processing 137 of 153 on page 3 - (1000 per page)\n", + "Processing 138 of 153 on page 3 - (1000 per page)\n", + "Processing 139 of 153 on page 3 - (1000 per page)\n", + "Processing 140 of 153 on page 3 - (1000 per page)\n", + "Processing 141 of 153 on page 3 - (1000 per page)\n", + "Processing 142 of 153 on page 3 - (1000 per page)\n", + "Processing 143 of 153 on page 3 - (1000 per page)\n", + "Processing 144 of 153 on page 3 - (1000 per page)\n", + "Processing 145 of 153 on page 3 - (1000 per page)\n", + "Processing 146 of 153 on page 3 - (1000 per page)\n", + "Processing 147 of 153 on page 3 - (1000 per page)\n", + "Processing 148 of 153 on page 3 - (1000 per page)\n", + "Processing 149 of 153 on page 3 - (1000 per page)\n", + "Processing 150 of 153 on page 3 - (1000 per page)\n", + "Processing 151 of 153 on page 3 - (1000 per page)\n", + "Processing 152 of 153 on page 3 - (1000 per page)\n", + "Processing 153 of 153 on page 3 - (1000 per page)\n", + "usgs_elev_tables load completed\n" + ] + } + ], "source": [ "\n", "# Has appx 2,150 HUCs to process, but this section goes quickly.\n", @@ -1092,45 +3349,44 @@ " 'Delimiter': '/'}\n", "page_iterator = paginator.paginate(**operation_parameters)\n", "page_count = 0\n", - "num_pages = len(page_iterator)\n", - "print(f\"num of pages is {num_pages}\")\n", - "# for page in page_iterator:\n", - " \n", - "# prefix_objects = page['CommonPrefixes']\n", - "# for i, prefix_obj in enumerate(prefix_objects):\n", - "# display_index = (i + 1) + (1000 * page_count)\n", - "# print(f\"Processing {i+1} of\"\n", - "# f\" {len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\")\n", - "# huc_prefix = prefix_obj.get(\"Prefix\")\n", - "# usgs_elev_table_key = f'{huc_prefix}usgs_elev_table.csv'\n", - "# try:\n", - "# uet = S3_CLIENT.get_object(\n", - "# Bucket=FIM_BUCKET, \n", - "# Key=usgs_elev_table_key\n", - "# )['Body']\n", - "# uet_df = pd.read_csv(uet, header=0, usecols=uet_usecols)\n", - "# uet_df['fim_version'] = PUBLIC_FIM_VERSION\n", - "# uet_df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", - "# uet_df.to_sql(\n", - "# con=VIZ_DB_ENGINE,\n", - "# dtype={\n", - "# \"location_id\": Text(),\n", - "# \"nws_data_huc\": Text()\n", - "# },\n", - "# schema='derived',\n", - "# name='usgs_elev_table',\n", - "# index=False, \n", - "# if_exists='append'\n", - "# )\n", - "# except Exception as e:\n", - "# if \"NoSuchKey\" in str(e):\n", - "# pass\n", - "# else:\n", - "# raise e\n", - "\n", - "# page_count += 1\n", - " \n", - " \n", + "# num_pages = len(page_iterator) pagintors don't have a length parameter\n", + "#print(f\"num of pages is {num_pages}\")\n", + "for page in page_iterator:\n", + "\n", + " prefix_objects = page['CommonPrefixes']\n", + " for i, prefix_obj in enumerate(prefix_objects):\n", + " display_index = (i + 1) + (1000 * page_count)\n", + " print(f\"Processing {i+1} of\"\n", + " f\" {len(prefix_objects)} on page {page_count + 1} - (1000 per page)\")\n", + " huc_prefix = prefix_obj.get(\"Prefix\")\n", + " usgs_elev_table_key = f'{huc_prefix}usgs_elev_table.csv'\n", + " try:\n", + " uet = S3_CLIENT.get_object(\n", + " Bucket=FIM_BUCKET, \n", + " Key=usgs_elev_table_key\n", + " )['Body']\n", + " uet_df = pd.read_csv(uet, header=0, usecols=uet_usecols)\n", + " # uet_df['fim_version'] = PUBLIC_FIM_VERSION\n", + " uet_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", + " uet_df.to_sql(\n", + " con=VIZ_DB_ENGINE,\n", + " dtype={\n", + " \"location_id\": Text(),\n", + " \"nws_data_huc\": Text()\n", + " },\n", + " schema='derived',\n", + " name='usgs_elev_table',\n", + " index=False, \n", + " if_exists='append'\n", + " )\n", + " except Exception as e:\n", + " if \"NoSuchKey\" in str(e):\n", + " pass\n", + " else:\n", + " raise e\n", + "\n", + " page_count += 1\n", + "\n", "print(\"usgs_elev_tables load completed\")\n" ] }, @@ -1174,12 +3430,9 @@ "\n", " prefix_objects = page['CommonPrefixes']\n", " for i, prefix_obj in enumerate(prefix_objects):\n", - " \n", - " print(f\"Processing {(i+1) + (1000 * page_count)} of\"\n", - " f\"{len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\") \n", - " \n", + "\n", " print(f\"Processing {(i+1) + (1000 * page_count)} of\"\n", - " f\" {len(prefix_objects) * page_count} on page {page_count + 1} (1000 per page)\")\n", + " f\" {len(prefix_objects)} on page {page_count + 1} - (1000 per page)\")\n", " branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", " branch_files_result = S3_CLIENT.list_objects(\n", " Bucket=FIM_BUCKET, \n", @@ -1203,8 +3456,8 @@ " # print(\"...Reading with pandas...\")\n", " ht_df = pd.read_csv(ht, header=0, usecols=ht_usecols)\n", " # print('...Writing to db...')\n", - " ht_df['fim_version'] = PUBLIC_FIM_VERSION\n", - " ht_df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", + " # ht_df['fim_version'] = PUBLIC_FIM_VERSION\n", + " ht_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", " ht_df.to_sql(\n", " con=VIZ_DB_ENGINE, \n", " schema='derived',\n", From bc1c399d30f83e64f9f24928931b471873f676da Mon Sep 17 00:00:00 2001 From: RobH Date: Tue, 19 Nov 2024 12:21:06 +0000 Subject: [PATCH 12/18] WIP loading --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 517 ++++++++++-------- 1 file changed, 304 insertions(+), 213 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 3f5835fc..af2a3b51 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -224,6 +224,10 @@ "execution_count": 5, "id": "c96a49f2", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, "tags": [] }, "outputs": [ @@ -451,8 +455,7 @@ "metadata": { "collapsed": true, "jupyter": { - "outputs_hidden": true, - "source_hidden": true + "outputs_hidden": true }, "tags": [] }, @@ -525,8 +528,7 @@ "metadata": { "collapsed": true, "jupyter": { - "outputs_hidden": true, - "source_hidden": true + "outputs_hidden": true }, "tags": [] }, @@ -566,8 +568,7 @@ "metadata": { "collapsed": true, "jupyter": { - "outputs_hidden": true, - "source_hidden": true + "outputs_hidden": true }, "tags": [] }, @@ -606,8 +607,7 @@ "metadata": { "collapsed": true, "jupyter": { - "outputs_hidden": true, - "source_hidden": true + "outputs_hidden": true }, "tags": [] }, @@ -646,8 +646,7 @@ "metadata": { "collapsed": true, "jupyter": { - "outputs_hidden": true, - "source_hidden": true + "outputs_hidden": true }, "tags": [] }, @@ -686,8 +685,7 @@ "metadata": { "collapsed": true, "jupyter": { - "outputs_hidden": true, - "source_hidden": true + "outputs_hidden": true }, "tags": [] }, @@ -726,8 +724,7 @@ "metadata": { "collapsed": true, "jupyter": { - "outputs_hidden": true, - "source_hidden": true + "outputs_hidden": true }, "tags": [] }, @@ -915,9 +912,6 @@ "execution_count": 39, "id": "c77d6e5b-f5fb-422f-a6c3-0f53743cd631", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [ @@ -992,6 +986,9 @@ "execution_count": 43, "id": "3ea2a5be-9936-4abe-832a-8582f3c3af64", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -1012,7 +1009,7 @@ " fim_catchments.catchments_gms_branches_prvi;\n", "''', db_type=\"egis\")\n", "\n", - "print(\"Catchment Truncation for GMS (Level Path) Branchs Done\")" + "print(\"Catchment Truncation for GMS (Level Path) Branches Done\")" ] }, { @@ -1020,6 +1017,9 @@ "execution_count": 45, "id": "9ae8917f", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [ @@ -1033,108 +1033,6 @@ } ], "source": [ - "# pipeline_input = {\n", - "# \"configuration\": \"reference\",\n", - "# \"job_type\": \"auto\",\n", - "# \"data_type\": \"channel\",\n", - "# \"keep_raw\": False,\n", - "# \"reference_time\": datetime.now().strftime('%Y-%m-%d 00:00:00'),\n", - "# \"configuration_data_flow\": {\n", - "# \"db_max_flows\": [],\n", - "# \"db_ingest_groups\": [],\n", - "# \"python_preprocessing\": []\n", - "# },\n", - "# \"pipeline_products\": [\n", - "# {\n", - "# \"product\": \"static_hand_catchments_gms_branches\",\n", - "# \"configuration\": \"reference\",\n", - "# \"product_type\": \"fim\",\n", - "# \"run\": True,\n", - "# \"fim_configs\": [\n", - "# {\n", - "# \"name\": \"catchments_gms_branches\",\n", - "# \"target_table\": \"fim_catchments.catchments_gms_branches\",\n", - "# \"fim_type\": \"hand\",\n", - "# \"sql_file\": \"catchments_gms_branches\"\n", - "# }\n", - "# ],\n", - "# \"services\": [\n", - "# \"static_hand_catchments_gms_branches_noaa\"\n", - "# ],\n", - "# \"raster_outputs\": {\n", - "# \"output_bucket\": \"\",\n", - "# \"output_raster_workspaces\": []\n", - "# },\n", - "# \"postprocess_sql\": [],\n", - "# \"product_summaries\": [],\n", - "# \"python_preprocesing_dependent\": False\n", - "# },\n", - "# {\n", - "# \"product\": \"static_hand_catchments_gms_branches_hi\",\n", - "# \"configuration\": \"reference\",\n", - "# \"product_type\": \"fim\",\n", - "# \"run\": True,\n", - "# \"fim_configs\": [\n", - "# {\n", - "# \"name\": \"catchments_gms_branches_hi\",\n", - "# \"target_table\": \"fim_catchments.catchments_gms_branches_hi\",\n", - "# \"fim_type\": \"hand\",\n", - "# \"sql_file\": \"catchments_gms_branches_hi\"\n", - "# }\n", - "# ],\n", - "# \"services\": [\n", - "# \"static_hand_catchments_gms_branches_hi_noaa\"\n", - "# ],\n", - "# \"raster_outputs\": {\n", - "# \"output_bucket\": \"\",\n", - "# \"output_raster_workspaces\": []\n", - "# },\n", - "# \"postprocess_sql\": [],\n", - "# \"product_summaries\": [],\n", - "# \"python_preprocesing_dependent\": False\n", - "# },\n", - "# {\n", - "# \"product\": \"static_hand_catchments_gms_branches_prvi\",\n", - "# \"configuration\": \"reference\",\n", - "# \"product_type\": \"fim\",\n", - "# \"run\": True,\n", - "# \"fim_configs\": [\n", - "# {\n", - "# \"name\": \"catchments_gms_branches_prvi\",\n", - "# \"target_table\": \"fim_catchments.catchments_gms_branches_prvi\",\n", - "# \"fim_type\": \"hand\",\n", - "# \"sql_file\": \"catchments_gms_branches_prvi\"\n", - "# }\n", - "# ],\n", - "# \"services\": [\n", - "# \"static_hand_catchments_gms_branches_prvi_noaa\"\n", - "# ],\n", - "# \"raster_outputs\": {\n", - "# \"output_bucket\": \"\",\n", - "# \"output_raster_workspaces\": []\n", - "# },\n", - "# \"postprocess_sql\": [],\n", - "# \"product_summaries\": [],\n", - "# \"python_preprocesing_dependent\": False\n", - "# }\n", - "# ],\n", - "# \"sql_rename_dict\": {},\n", - "# \"logging_info\": {\n", - "# \"Timestamp\": int(datetime.now().timestamp())\n", - "# }\n", - "# }\n", - "\n", - "\n", - "\n", - "# STEPFUNCTION_CLIENT.start_execution(\n", - "# stateMachineArn = PIPELINE_ARN,\n", - "# name = pipeline_name,\n", - "# input= json.dumps(pipeline_input)\n", - "# )\n", - "\n", - "# print(\"Catchments GMS Branches (Level Paths / non branch 0) load kicked off.\"\n", - "# f\" Last runtime: 24:45.150. Pipeline : hv-vpp-ti-viz-pipeline - {pipeline_name}\")\n", - "\n", "\n", "pipeline_name = f\"hv_ti_data_loads_catchments_gms_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "pipeline_input = get_catchment_pipepline_input(\"gms\")\n", @@ -1163,7 +1061,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 59, "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", "metadata": { "collapsed": true, @@ -3405,7 +3303,183 @@ "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hydrotable reloaded - started\n", + "Processing 1 of 1000 on page 1 - (1000 per page)\n", + "Processing 2 of 1000 on page 2 - (1000 per page)\n", + "Processing 3 of 1000 on page 3 - (1000 per page)\n", + "Processing 4 of 1000 on page 4 - (1000 per page)\n", + "Processing 5 of 1000 on page 5 - (1000 per page)\n", + "Processing 6 of 1000 on page 6 - (1000 per page)\n", + "Processing 7 of 1000 on page 7 - (1000 per page)\n", + "Processing 8 of 1000 on page 8 - (1000 per page)\n", + "Processing 9 of 1000 on page 9 - (1000 per page)\n", + "Processing 10 of 1000 on page 10 - (1000 per page)\n", + "Processing 11 of 1000 on page 11 - (1000 per page)\n", + "Processing 12 of 1000 on page 12 - (1000 per page)\n", + "Processing 13 of 1000 on page 13 - (1000 per page)\n", + "Processing 14 of 1000 on page 14 - (1000 per page)\n", + "Processing 15 of 1000 on page 15 - (1000 per page)\n", + "Processing 16 of 1000 on page 16 - (1000 per page)\n", + "Processing 17 of 1000 on page 17 - (1000 per page)\n", + "Processing 18 of 1000 on page 18 - (1000 per page)\n", + "Processing 19 of 1000 on page 19 - (1000 per page)\n", + "Processing 20 of 1000 on page 20 - (1000 per page)\n", + "Processing 21 of 1000 on page 21 - (1000 per page)\n", + "Processing 22 of 1000 on page 22 - (1000 per page)\n", + "Processing 23 of 1000 on page 23 - (1000 per page)\n", + "Processing 24 of 1000 on page 24 - (1000 per page)\n", + "Processing 25 of 1000 on page 25 - (1000 per page)\n", + "Processing 26 of 1000 on page 26 - (1000 per page)\n", + "Processing 27 of 1000 on page 27 - (1000 per page)\n", + "Processing 28 of 1000 on page 28 - (1000 per page)\n", + "Processing 29 of 1000 on page 29 - (1000 per page)\n", + "Processing 30 of 1000 on page 30 - (1000 per page)\n", + "Processing 31 of 1000 on page 31 - (1000 per page)\n", + "Processing 32 of 1000 on page 32 - (1000 per page)\n", + "Processing 33 of 1000 on page 33 - (1000 per page)\n", + "Processing 34 of 1000 on page 34 - (1000 per page)\n", + "Processing 35 of 1000 on page 35 - (1000 per page)\n", + "Processing 36 of 1000 on page 36 - (1000 per page)\n", + "Processing 37 of 1000 on page 37 - (1000 per page)\n", + "Processing 38 of 1000 on page 38 - (1000 per page)\n", + "Processing 39 of 1000 on page 39 - (1000 per page)\n", + "Processing 40 of 1000 on page 40 - (1000 per page)\n", + "Processing 41 of 1000 on page 41 - (1000 per page)\n", + "Processing 42 of 1000 on page 42 - (1000 per page)\n", + "Processing 43 of 1000 on page 43 - (1000 per page)\n", + "Processing 44 of 1000 on page 44 - (1000 per page)\n", + "Processing 45 of 1000 on page 45 - (1000 per page)\n", + "Processing 46 of 1000 on page 46 - (1000 per page)\n", + "Processing 47 of 1000 on page 47 - (1000 per page)\n", + "Processing 48 of 1000 on page 48 - (1000 per page)\n", + "Processing 49 of 1000 on page 49 - (1000 per page)\n", + "Processing 50 of 1000 on page 50 - (1000 per page)\n", + "Processing 51 of 1000 on page 51 - (1000 per page)\n", + "Processing 52 of 1000 on page 52 - (1000 per page)\n", + "Processing 53 of 1000 on page 53 - (1000 per page)\n", + "Processing 54 of 1000 on page 54 - (1000 per page)\n", + "Processing 55 of 1000 on page 55 - (1000 per page)\n", + "Processing 56 of 1000 on page 56 - (1000 per page)\n", + "Processing 57 of 1000 on page 57 - (1000 per page)\n", + "Processing 58 of 1000 on page 58 - (1000 per page)\n", + "Processing 59 of 1000 on page 59 - (1000 per page)\n", + "Processing 60 of 1000 on page 60 - (1000 per page)\n", + "Processing 61 of 1000 on page 61 - (1000 per page)\n", + "Processing 62 of 1000 on page 62 - (1000 per page)\n", + "Processing 63 of 1000 on page 63 - (1000 per page)\n", + "Processing 64 of 1000 on page 64 - (1000 per page)\n", + "Processing 65 of 1000 on page 65 - (1000 per page)\n", + "Processing 66 of 1000 on page 66 - (1000 per page)\n", + "Processing 67 of 1000 on page 67 - (1000 per page)\n", + "Processing 68 of 1000 on page 68 - (1000 per page)\n", + "Processing 69 of 1000 on page 69 - (1000 per page)\n", + "Processing 70 of 1000 on page 70 - (1000 per page)\n", + "Processing 71 of 1000 on page 71 - (1000 per page)\n", + "Processing 72 of 1000 on page 72 - (1000 per page)\n", + "Processing 73 of 1000 on page 73 - (1000 per page)\n", + "Processing 74 of 1000 on page 74 - (1000 per page)\n", + "Processing 75 of 1000 on page 75 - (1000 per page)\n", + "Processing 76 of 1000 on page 76 - (1000 per page)\n", + "Processing 77 of 1000 on page 77 - (1000 per page)\n", + "Processing 78 of 1000 on page 78 - (1000 per page)\n", + "Processing 79 of 1000 on page 79 - (1000 per page)\n", + "Processing 80 of 1000 on page 80 - (1000 per page)\n", + "Processing 81 of 1000 on page 81 - (1000 per page)\n", + "Processing 82 of 1000 on page 82 - (1000 per page)\n", + "Processing 83 of 1000 on page 83 - (1000 per page)\n", + "Processing 84 of 1000 on page 84 - (1000 per page)\n", + "Processing 85 of 1000 on page 85 - (1000 per page)\n", + "Processing 86 of 1000 on page 86 - (1000 per page)\n", + "Processing 87 of 1000 on page 87 - (1000 per page)\n", + "Processing 88 of 1000 on page 88 - (1000 per page)\n", + "Processing 89 of 1000 on page 89 - (1000 per page)\n", + "Processing 90 of 1000 on page 90 - (1000 per page)\n", + "Processing 91 of 1000 on page 91 - (1000 per page)\n", + "Processing 92 of 1000 on page 92 - (1000 per page)\n", + "Processing 93 of 1000 on page 93 - (1000 per page)\n", + "Processing 94 of 1000 on page 94 - (1000 per page)\n", + "Processing 95 of 1000 on page 95 - (1000 per page)\n", + "Processing 96 of 1000 on page 96 - (1000 per page)\n", + "Processing 97 of 1000 on page 97 - (1000 per page)\n", + "Processing 98 of 1000 on page 98 - (1000 per page)\n", + "Processing 99 of 1000 on page 99 - (1000 per page)\n", + "Processing 100 of 1000 on page 100 - (1000 per page)\n", + "Processing 101 of 1000 on page 101 - (1000 per page)\n", + "Processing 102 of 1000 on page 102 - (1000 per page)\n", + "Processing 103 of 1000 on page 103 - (1000 per page)\n", + "Processing 104 of 1000 on page 104 - (1000 per page)\n", + "Processing 105 of 1000 on page 105 - (1000 per page)\n", + "Processing 106 of 1000 on page 106 - (1000 per page)\n", + "Processing 107 of 1000 on page 107 - (1000 per page)\n", + "Processing 108 of 1000 on page 108 - (1000 per page)\n", + "Processing 109 of 1000 on page 109 - (1000 per page)\n", + "Processing 110 of 1000 on page 110 - (1000 per page)\n", + "Processing 111 of 1000 on page 111 - (1000 per page)\n", + "Processing 112 of 1000 on page 112 - (1000 per page)\n", + "Processing 113 of 1000 on page 113 - (1000 per page)\n", + "Processing 114 of 1000 on page 114 - (1000 per page)\n", + "Processing 115 of 1000 on page 115 - (1000 per page)\n", + "Processing 116 of 1000 on page 116 - (1000 per page)\n", + "Processing 117 of 1000 on page 117 - (1000 per page)\n", + "Processing 118 of 1000 on page 118 - (1000 per page)\n", + "Processing 119 of 1000 on page 119 - (1000 per page)\n", + "Processing 120 of 1000 on page 120 - (1000 per page)\n", + "Processing 121 of 1000 on page 121 - (1000 per page)\n", + "Processing 122 of 1000 on page 122 - (1000 per page)\n", + "Processing 123 of 1000 on page 123 - (1000 per page)\n", + "Processing 124 of 1000 on page 124 - (1000 per page)\n", + "Processing 125 of 1000 on page 125 - (1000 per page)\n", + "Processing 126 of 1000 on page 126 - (1000 per page)\n", + "Processing 127 of 1000 on page 127 - (1000 per page)\n", + "Processing 128 of 1000 on page 128 - (1000 per page)\n", + "Processing 129 of 1000 on page 129 - (1000 per page)\n", + "Processing 130 of 1000 on page 130 - (1000 per page)\n", + "Processing 131 of 1000 on page 131 - (1000 per page)\n", + "Processing 132 of 1000 on page 132 - (1000 per page)\n", + "Processing 133 of 1000 on page 133 - (1000 per page)\n", + "Processing 134 of 1000 on page 134 - (1000 per page)\n", + "Processing 135 of 1000 on page 135 - (1000 per page)\n", + "Processing 136 of 1000 on page 136 - (1000 per page)\n", + "Processing 137 of 1000 on page 137 - (1000 per page)\n", + "Processing 138 of 1000 on page 138 - (1000 per page)\n", + "Processing 139 of 1000 on page 139 - (1000 per page)\n", + "Processing 140 of 1000 on page 140 - (1000 per page)\n", + "Processing 141 of 1000 on page 141 - (1000 per page)\n", + "Processing 142 of 1000 on page 142 - (1000 per page)\n", + "Processing 143 of 1000 on page 143 - (1000 per page)\n", + "Processing 144 of 1000 on page 144 - (1000 per page)\n", + "Processing 145 of 1000 on page 145 - (1000 per page)\n", + "Processing 146 of 1000 on page 146 - (1000 per page)\n", + "Processing 147 of 1000 on page 147 - (1000 per page)\n", + "Processing 148 of 1000 on page 148 - (1000 per page)\n", + "Processing 149 of 1000 on page 149 - (1000 per page)\n", + "Processing 150 of 1000 on page 150 - (1000 per page)\n", + "Processing 151 of 1000 on page 151 - (1000 per page)\n", + "Processing 152 of 1000 on page 152 - (1000 per page)\n", + "Processing 153 of 1000 on page 153 - (1000 per page)\n", + "Processing 154 of 1000 on page 154 - (1000 per page)\n", + "Processing 155 of 1000 on page 155 - (1000 per page)\n", + "Processing 156 of 1000 on page 156 - (1000 per page)\n", + "Processing 157 of 1000 on page 157 - (1000 per page)\n", + "Processing 158 of 1000 on page 158 - (1000 per page)\n", + "Processing 159 of 1000 on page 159 - (1000 per page)\n", + "Processing 160 of 1000 on page 160 - (1000 per page)\n", + "Processing 161 of 1000 on page 161 - (1000 per page)\n", + "Processing 162 of 1000 on page 162 - (1000 per page)\n", + "Processing 163 of 1000 on page 163 - (1000 per page)\n", + "Processing 164 of 1000 on page 164 - (1000 per page)\n", + "Processing 165 of 1000 on page 165 - (1000 per page)\n", + "Processing 166 of 1000 on page 166 - (1000 per page)\n", + "Processing 167 of 1000 on page 167 - (1000 per page)\n", + "Processing 168 of 1000 on page 168 - (1000 per page)\n" + ] + } + ], "source": [ "\n", "# Takes appx 5.75 to 6 hrs to run\n", @@ -3431,8 +3505,9 @@ " prefix_objects = page['CommonPrefixes']\n", " for i, prefix_obj in enumerate(prefix_objects):\n", "\n", - " print(f\"Processing {(i+1) + (1000 * page_count)} of\"\n", + " print(f\"Processing {i+1} of\"\n", " f\" {len(prefix_objects)} on page {page_count + 1} - (1000 per page)\")\n", + "\n", " branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", " branch_files_result = S3_CLIENT.list_objects(\n", " Bucket=FIM_BUCKET, \n", @@ -3468,10 +3543,9 @@ " except Exception as e:\n", " raise e\n", " print(f'Fetch failed: {e}')\n", - " \n", + "\n", " page_count += 1\n", - " \n", - " \n", + "\n", "end_dt = datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(\"hydrotable reload done\")\n", @@ -3709,44 +3783,78 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "id": "9a249854-b9cd-4ccb-afea-96c35708515d", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done dropping src_skill and src_skill_temp tables\n" + ] + } + ], "source": [ "# Prep the dbs for the new load\n", - "#sf.execute_sql('DROP TABLE IF EXISTS derived.src_skill_temp;')\n", - "#sf.execute_sql('DROP TABLE IF EXISTS reference.src_skill;', db_type='egis')\n", + "#sf.eecute_sql('DROP TABLE IF EXISTS derived.src_skill_temp;', db_type='viz')\n", "#print(\"Done dropping src_skill and src_skill_temp tables\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "id": "c24f509a-90df-492a-a0d4-aa9eab8e7041", "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading file...\n", + ".. Downloading: s3://hydrovis-ti-deployment-us-east-1/fim/hand_4_5_11_1/qa_datasets/agg_nwm_recurr_flow_elev_stats_location_id.csv\n", + "\n", + ".. 1 files downloaded and loaded into the dataframe\n", + "File read. 2967 records found\n", + "Done loading derived.src_skill_temp table\n", + "... duration was 0:00:00\n" + ] + } + ], "source": [ "\n", "# Load the src_skill_temp table\n", "start_dt = datetime.now()\n", "\n", - "event = {\n", - " 'target_table': 'derived.src_skill_temp',\n", - " 'target_cols': None, # This means \"all\"\n", - " 'file': f'{QA_DATASETS_DPATH}/agg_nwm_recurr_flow_elev_stats_location_id.csv',\n", - " 'bucket': FIM_BUCKET,\n", - " 'reference_time': '2023-08-23 00:00:00',\n", - " 'keep_flows_at_or_above': 0,\n", - " 'iteration_index': 0,\n", - " 'db_type': 'viz'\n", - "}\n", + "sf.eecute_sql('DROP TABLE IF EXISTS derived.src_skill_temp;', db_type='viz')\n", + "\n", + "file_handle = 'agg_nwm_recurr_flow_elev_stats_location_id.csv'\n", + "\n", + "print(\"Reading file...\")\n", + "# df = pd.read_csv(local_download_path)\n", + "file_to_download = f\"{QA_DATASETS_DPATH}/{file_handle}\"\n", + "df = s3_sf.download_S3_csv_files_to_df_from_list(FIM_BUCKET, [file_to_download], True)\n", + "print(f\"File read. {len(df)} records found\")\n", + "\n", + "db_type = \"viz\"\n", + "db_engine = sf.get_db_engine(db_type)\n", + "\n", + "df.to_sql(\n", + " name='src_skill_temp',\n", + " con=db_engine,\n", + " schema='derived',\n", + " if_exists='replace',\n", + " index=False\n", + ")\n", "\n", - "execute_db_ingest(event, None)\n", "print(\"Done loading derived.src_skill_temp table\")\n", "end_dt = datetime.now()\n", "time_duration = end_dt - start_dt\n", @@ -3755,12 +3863,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "id": "b5639447-9a5a-4add-9c36-ebd945a5a8d0", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done loading derived.src_skill table\n", + "... duration was 0:00:00\n" + ] + } + ], "source": [ "\n", "# Load into src_skill table adding geometry to it from external.usgs_gage. Yes.. more/less straight from WRDS tables\n", @@ -3768,24 +3885,23 @@ "\n", "start_dt = datetime.now()\n", "\n", - "sf.execute_sql('DROP TABLE IF EXISTS derived.src_skill;')\n", + "sf.execute_sql('DROP TABLE IF EXISTS derived.src_skill;', db_type='viz')\n", "\n", - "sql = f\"\"\"\n", + "sql = f'''\n", "SELECT\n", - "\t(row_number() OVER ())::int as oid,\n", - "\tgage.name,\n", - "\tLPAD(skill.location_id::text, 8, '0') as location_id,\n", - "\tskill.nrmse,\n", - "\tskill.mean_abs_y_diff_ft,\n", - "\tskill.mean_y_diff_ft,\n", - "\tskill.percent_bias,\n", - " '{PUBLIC_FIM_VERSION}' as {COLUMN_NAME_FIM_VERSION},\n", - " '{FIM_MODEL_VERSION}' as {COLUMN_NAME_MODEL_VERSION},\n", - "\tgage.geo_point as geom\n", + " (row_number() OVER ())::int as oid,\n", + " gage.name,\n", + " LPAD(skill_temp.location_id::text, 8, '0') as location_id,\n", + " skill_temp.nrmse,\n", + " skill_temp.mean_abs_y_diff_ft,\n", + " skill_temp.mean_y_diff_ft,\n", + " skill_temp.percent_bias,\n", + " '{HAND_MODEL_VERSION}' as {COLUMN_NAME_MODEL_VERSION},\n", + " gage.geo_point as geom\n", "INTO derived.src_skill\n", - "FROM derived.src_skill_temp skill\n", - "JOIN external.usgs_gage AS gage ON LPAD(gage.usgs_gage_id::text, 8, '0') = LPAD(skill.location_id::text, 8, '0')\n", - "\"\"\"\n", + "FROM derived.src_skill_temp skill_temp\n", + "JOIN external.usgs_gage AS gage ON LPAD(gage.usgs_gage_id::text, 8, '0') = LPAD(skill_temp.location_id::text, 8, '0')\n", + "'''\n", "\n", "sf.execute_sql(sql)\n", "\n", @@ -3808,12 +3924,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "id": "0df23345-47a4-490a-b10f-2020e3a2a68e", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Successfully copied derived.src_skill from the VIZ DB to reference.src_skill in the EGIS DB!\n", + "Done\n" + ] + } + ], "source": [ "\n", "sf.move_data_from_viz_to_egis(\"derived.src_skill\", \"reference.src_skill\")\n", @@ -3832,63 +3957,29 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "b291a461-e36f-4049-84b6-dd59cd1c4ad2", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "\n", - "# Make copies of current dbs for 4.4.0.0 (4.5.2.11)\n", - "# DONE: for 4.4.0.0 (4.5.2.11)\n", - "\n", - "# NOTE: Aug 2024: The problem with not droppign them and rebuilding them with indexes, is that if the table schema\n", - "# changes it is not reflected\n", - "\n", - "\n", - "# Points\n", - "new_table_name = f\"reference.fim_performance_points_{OLD_FIM_TAG}\"\n", - "sql = f'''\n", - " CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.fim_performance_points;\n", - "'''\n", - "sf.execute_sql(sql, db_type='egis')\n", - "print(f\"fim_performance_points copied to {new_table_name} if it does not already exists\")\n", - "\n", - "\n", - "# Catchments\n", - "new_table_name = f\"reference.fim_performance_catchments_{OLD_FIM_TAG}\"\n", - "sql = f'''\n", - " CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.fim_performance_catchments;\n", - "'''\n", - "sf.execute_sql(sql, db_type='egis')\n", - "print(f\"fim_performance_catchments copied to {new_table_name} if it does not already exists\")\n", - "\n", - "\n", - "# Polys\n", - "new_table_name = f\"reference.fim_performance_polys_{OLD_FIM_TAG}\"\n", - "sql = f'''\n", - " CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE reference.fim_performance_polys;\n", - "'''\n", - "sf.execute_sql(sql, db_type='egis')\n", - "print(f\"fim_performance_polys copied to {new_table_name} if it does not already exists\")\n", - "\n", - "print(\"Done making backups of the FIM performance tables\")\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 58, "id": "f4fcdfab-3936-4a0a-8ab9-605e9a64f9bd", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "syntax error at or near \"EXISTS\"\nLINE 1: TRUNCATE TABLE IF EXISTS reference.fim_performance_points\n ^\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mSyntaxError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[58], line 14\u001b[0m\n\u001b[1;32m 12\u001b[0m sql \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTRUNCATE TABLE IF EXISTS \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtb_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;66;03m# print(sql)\u001b[39;00m\n\u001b[0;32m---> 14\u001b[0m \u001b[43msf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute_sql\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m,\u001b[49m\u001b[43mdb_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43megis\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAll fim_performance tables trunated if they exist\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/SageMaker/rob.hanna/code/ti_add_upload_script_hand_4_5_11_1/Core/Manual_Workflows/FIM_data_loads/../helper_functions/shared_functions.py:215\u001b[0m, in \u001b[0;36mexecute_sql\u001b[0;34m(sql, db_type)\u001b[0m\n\u001b[1;32m 213\u001b[0m db_connection\u001b[38;5;241m.\u001b[39mcommit()\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 215\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 217\u001b[0m db_connection\u001b[38;5;241m.\u001b[39mclose()\n", + "File \u001b[0;32m~/SageMaker/rob.hanna/code/ti_add_upload_script_hand_4_5_11_1/Core/Manual_Workflows/FIM_data_loads/../helper_functions/shared_functions.py:212\u001b[0m, in \u001b[0;36mexecute_sql\u001b[0;34m(sql, db_type)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 211\u001b[0m cur \u001b[38;5;241m=\u001b[39m db_connection\u001b[38;5;241m.\u001b[39mcursor()\n\u001b[0;32m--> 212\u001b[0m \u001b[43mcur\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 213\u001b[0m db_connection\u001b[38;5;241m.\u001b[39mcommit()\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "\u001b[0;31mSyntaxError\u001b[0m: syntax error at or near \"EXISTS\"\nLINE 1: TRUNCATE TABLE IF EXISTS reference.fim_performance_points\n ^\n" + ] + } + ], "source": [ "# clean up tables for new load\n", "\n", - "# TODO: Aug 2024: Add postgresql if / else. Truncate \"if exists\" doesn't exist. :)\n", "\n", "table_names = [\n", " \"reference.fim_performance_points\",\n", @@ -4520,7 +4611,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.10.15" } }, "nbformat": 4, From b66f929d2f661a1060beb5fe220aa6c8cba17fba Mon Sep 17 00:00:00 2001 From: RobH Date: Sun, 24 Nov 2024 00:58:29 +0000 Subject: [PATCH 13/18] wip --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 3179 +++-------------- .../helper_functions/s3_shared_functions.py | 50 +- 2 files changed, 529 insertions(+), 2700 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index af2a3b51..f446dc49 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -29,35 +29,74 @@ "name": "stdout", "output_type": "stream", "text": [ + "Requirement already satisfied: geopandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.0.1)\n", + "Requirement already satisfied: numpy>=1.22 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (1.26.4)\n", + "Requirement already satisfied: pyogrio>=0.7.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (0.10.0)\n", + "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (24.2)\n", + "Requirement already satisfied: pandas>=1.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.2.3)\n", + "Requirement already satisfied: pyproj>=3.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (3.7.0)\n", + "Requirement already satisfied: shapely>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.0.6)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2.9.0)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.2)\n", + "Requirement already satisfied: certifi in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pyogrio>=0.7.2->geopandas) (2024.8.30)\n", + "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=1.4.0->geopandas) (1.16.0)\n", + "Requirement already satisfied: xarray in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (2024.10.0)\n", + "Requirement already satisfied: numpy>=1.24 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray) (1.26.4)\n", + "Requirement already satisfied: packaging>=23.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray) (24.2)\n", + "Requirement already satisfied: pandas>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray) (2.2.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray) (2.9.0)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray) (2024.2)\n", + "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=2.1->xarray) (1.16.0)\n", + "Requirement already satisfied: geoalchemy2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (0.16.0)\n", + "Requirement already satisfied: SQLAlchemy>=1.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (2.0.36)\n", + "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (24.2)\n", + "Requirement already satisfied: typing-extensions>=4.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (4.12.2)\n", + "Requirement already satisfied: greenlet!=0.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (3.1.1)\n", + "Requirement already satisfied: rioxarray in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (0.18.1)\n", + "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (24.2)\n", + "Requirement already satisfied: rasterio>=1.3.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (1.4.2)\n", + "Requirement already satisfied: xarray>=2024.7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (2024.10.0)\n", + "Requirement already satisfied: pyproj>=3.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (3.7.0)\n", + "Requirement already satisfied: numpy>=1.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (1.26.4)\n", + "Requirement already satisfied: certifi in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pyproj>=3.3->rioxarray) (2024.8.30)\n", + "Requirement already satisfied: affine in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (2.4.0)\n", + "Requirement already satisfied: attrs in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (23.2.0)\n", + "Requirement already satisfied: click>=4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (8.1.7)\n", + "Requirement already satisfied: cligj>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (0.7.2)\n", + "Requirement already satisfied: click-plugins in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (1.1.1)\n", + "Requirement already satisfied: pyparsing in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (3.2.0)\n", + "Requirement already satisfied: pandas>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray>=2024.7.0->rioxarray) (2.2.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2.9.0)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2024.2)\n", + "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=2.1->xarray>=2024.7.0->rioxarray) (1.16.0)\n", + "Requirement already satisfied: s3fs in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (0.4.2)\n", + "Requirement already satisfied: botocore>=1.12.91 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from s3fs) (1.35.63)\n", + "Requirement already satisfied: fsspec>=0.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from s3fs) (2024.10.0)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (1.0.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (2.9.0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (2.2.3)\n", + "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil<3.0.0,>=2.1->botocore>=1.12.91->s3fs) (1.16.0)\n", "Requirement already satisfied: python-dotenv in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.0.1)\n", - "All loaded\n" + "*** All loaded ***\n" ] } ], "source": [ "# Cell to manually pip reload a packages that the Jupyter engine not retained\n", "# !pip install numpy\n", - "# !pip install geopandas\n", + "!pip install geopandas\n", "# !pip install pyarrow\n", - "# !pip install xarray\n", - "# !pip install geoalchemy2\n", + "!pip install xarray\n", + "!pip install geoalchemy2\n", "# !pip install contextily\n", - "# !pip install rioxarray\n", + "!pip install rioxarray\n", "\n", + "!pip install s3fs # to get a newer version, old one on system\n", "!pip install python-dotenv\n", - "print(\"All loaded\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "103a0d12-b2a0-4586-ba0f-0ed6b2c4eb89", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# pd.set_option(\"max_info_rows\", 100000) # override " + "print(\"*** All loaded ***\")\n" ] }, { @@ -85,6 +124,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "option set\n", "imports loaded\n" ] } @@ -123,6 +163,9 @@ "\n", "from helper_functions.viz_classes import database\n", "\n", + "pd.set_option(\"max_info_rows\", 1000000) # override \n", + "print(\"option set\")\n", + "\n", "print(\"imports loaded\")\n" ] }, @@ -165,7 +208,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 4, "id": "e944eff3-6023-48f4-9f57-27304a240447", "metadata": { "tags": [] @@ -191,7 +234,6 @@ "QA_DATASETS_DPATH = f\"{HAND_ROOT_DPATH}/qa_datasets\"\n", "\n", "FIM_BUCKET = \"hydrovis-ti-deployment-us-east-1\"\n", - "FIM_CROSSWALK_FPATH = os.path.join(HAND_DATASETS_DPATH, \"crosswalk_table.csv\")\n", "PIPELINE_ARN = 'arn:aws:states:us-east-1:526904826677:stateMachine:hv-vpp-ti-viz-pipeline'\n", "\n", "COLUMN_NAME_MODEL_VERSION = \"model_version\"\n", @@ -221,35 +263,18 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "c96a49f2", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Getting column name from fim/hand_4_5_11_1/hand_datasets/crosswalk_table.csv\n", - "(hand_id integer,feature_id bigint,huc8 TEXT,branch_id bigint,hydro_id integer,lake_id integer)\n", - "***> Established db connection to: hv-vpp-ti-viz-processing.c4vzypepnkx3.us-east-1.rds.amazonaws.com from ()\n", - "Deleting/Creating derived.fim4_featureid_crosswalk using columns (hand_id integer,feature_id bigint,huc8 TEXT,branch_id bigint,hydro_id integer,lake_id integer)\n", - "Importing fim/hand_4_5_11_1/hand_datasets/crosswalk_table.csv to derived.fim4_featureid_crosswalk\n", - "Adding model_version column to derived.fim4_featureid_crosswalk\n", - "Adding feature id index to derived.fim4_featureid_crosswalk\n", - "Adding hydro id index to derived.fim4_featureid_crosswalk\n", - "\n", - "... Estimated time to completion is just a few mins\n", - "Successully loaded derived.fim4_featureid_crosswalk and updated it\n" - ] - } - ], + "outputs": [], "source": [ + "\n", + "FIM_CROSSWALK_FPATH = os.path.join(HAND_DATASETS_DPATH, \"crosswalk_table.csv\")\n", "\n", "print(f\"Getting column name from {FIM_CROSSWALK_FPATH}\")\n", "\n", @@ -397,6 +422,9 @@ "execution_count": null, "id": "dfc0a44c-8b46-4db5-a9a0-c65e0bf1e20b", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -450,24 +478,15 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "6a698067", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "function: get_aep_pipeline_input loaded\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Aug 6, 2024: Note: This was created after all intervals were created, so only HW was tested against\n", @@ -523,25 +542,15 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "f6d6ee69", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "AEP : 2 year flows ie: rf_2_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_2_20241116T1902\n" - ] - } - ], + "outputs": [], "source": [ "\n", "#### 2 Year Flow\n", @@ -563,25 +572,15 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "a4f89d9a", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "AEP : 5 year flows ie: rf_5_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_5_20241116T1921\n" - ] - } - ], + "outputs": [], "source": [ "\n", "#### 5 Year Flow\n", @@ -602,25 +601,15 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "791d1a8b", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "AEP : 10 year flows ie: rf_10_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_10_20241116T1921\n" - ] - } - ], + "outputs": [], "source": [ "\n", "#### 10 Year Flow\n", @@ -641,25 +630,15 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "1bb87128", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "AEP : 25 year flows ie: rf_25_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_25_20241116T1921\n" - ] - } - ], + "outputs": [], "source": [ "\n", "#### 25 Year Flow\n", @@ -680,25 +659,15 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "4832e4e0", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "AEP : 50 year flows ie: rf_50_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_50_20241116T1921\n" - ] - } - ], + "outputs": [], "source": [ "\n", "#### 50 Year Flow\n", @@ -719,26 +688,15 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "187e83bc-ebbe-4615-a046-e0ef7b09ad3b", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "AEP : High Water year flows ie: rf_hw_inundation kicked off. Can take 15 - 45 mins.\n", - "Step Function Pipeline : hv-vpp-ti-viz-pipeline : Pipeline run name started: sagemaker_aep_hw_20241116T1921\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "\n", "#### HW (High Water) Flow\n", @@ -782,20 +740,15 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "a7495759", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "get_catchment_pipepline_input loaded\n" - ] - } - ], + "outputs": [], "source": [ "\n", "def get_catchment_pipepline_input(branch_key):\n", @@ -909,21 +862,15 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "id": "c77d6e5b-f5fb-422f-a6c3-0f53743cd631", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Catchment Truncation for Branch 0 Done\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Npte: The three db's here were renamed from:\n", @@ -942,20 +889,15 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "id": "8c2806b8-5b9f-435c-b0ae-a28f9224b938", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Catchments Branch 0 load kicked off. Last runtime: 23:38.019. Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - hv_ti_data_loads_catchments_branch_0_20241118T2321\n" - ] - } - ], + "outputs": [], "source": [ "\n", "pipeline_name = f\"hv_ti_data_loads_catchments_branch_0_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", @@ -983,7 +925,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "id": "3ea2a5be-9936-4abe-832a-8582f3c3af64", "metadata": { "jupyter": { @@ -991,15 +933,7 @@ }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Catchment Truncation for GMS (Level Path) Branchs Done\n" - ] - } - ], + "outputs": [], "source": [ "\n", "sf.execute_sql('''\n", @@ -1014,7 +948,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "id": "9ae8917f", "metadata": { "jupyter": { @@ -1022,16 +956,7 @@ }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Catchments GMS Branches (Level Paths / non branch 0) load kicked off. Takes appx 25 mins (depending on other processess)\n", - " Step Function Pipeline : hv-vpp-ti-viz-pipeline : Run Name - hv_ti_data_loads_catchments_gms_20241119T0255\n" - ] - } - ], + "outputs": [], "source": [ "\n", "pipeline_name = f\"hv_ti_data_loads_catchments_gms_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", @@ -1061,2178 +986,16 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true + "source_hidden": true }, "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Processing 1 of 1000 on page 1 - (1000 per page)\n", - "Processing 2 of 1000 on page 1 - (1000 per page)\n", - "Processing 3 of 1000 on page 1 - (1000 per page)\n", - "Processing 4 of 1000 on page 1 - (1000 per page)\n", - "Processing 5 of 1000 on page 1 - (1000 per page)\n", - "Processing 6 of 1000 on page 1 - (1000 per page)\n", - "Processing 7 of 1000 on page 1 - (1000 per page)\n", - "Processing 8 of 1000 on page 1 - (1000 per page)\n", - "Processing 9 of 1000 on page 1 - (1000 per page)\n", - "Processing 10 of 1000 on page 1 - (1000 per page)\n", - "Processing 11 of 1000 on page 1 - (1000 per page)\n", - "Processing 12 of 1000 on page 1 - (1000 per page)\n", - "Processing 13 of 1000 on page 1 - (1000 per page)\n", - "Processing 14 of 1000 on page 1 - (1000 per page)\n", - "Processing 15 of 1000 on page 1 - (1000 per page)\n", - "Processing 16 of 1000 on page 1 - (1000 per page)\n", - "Processing 17 of 1000 on page 1 - (1000 per page)\n", - "Processing 18 of 1000 on page 1 - (1000 per page)\n", - "Processing 19 of 1000 on page 1 - (1000 per page)\n", - "Processing 20 of 1000 on page 1 - (1000 per page)\n", - "Processing 21 of 1000 on page 1 - (1000 per page)\n", - "Processing 22 of 1000 on page 1 - (1000 per page)\n", - "Processing 23 of 1000 on page 1 - (1000 per page)\n", - "Processing 24 of 1000 on page 1 - (1000 per page)\n", - "Processing 25 of 1000 on page 1 - (1000 per page)\n", - "Processing 26 of 1000 on page 1 - (1000 per page)\n", - "Processing 27 of 1000 on page 1 - (1000 per page)\n", - "Processing 28 of 1000 on page 1 - (1000 per page)\n", - "Processing 29 of 1000 on page 1 - (1000 per page)\n", - "Processing 30 of 1000 on page 1 - (1000 per page)\n", - "Processing 31 of 1000 on page 1 - (1000 per page)\n", - "Processing 32 of 1000 on page 1 - (1000 per page)\n", - "Processing 33 of 1000 on page 1 - (1000 per page)\n", - "Processing 34 of 1000 on page 1 - (1000 per page)\n", - "Processing 35 of 1000 on page 1 - (1000 per page)\n", - "Processing 36 of 1000 on page 1 - (1000 per page)\n", - "Processing 37 of 1000 on page 1 - (1000 per page)\n", - "Processing 38 of 1000 on page 1 - (1000 per page)\n", - "Processing 39 of 1000 on page 1 - (1000 per page)\n", - "Processing 40 of 1000 on page 1 - (1000 per page)\n", - "Processing 41 of 1000 on page 1 - (1000 per page)\n", - "Processing 42 of 1000 on page 1 - (1000 per page)\n", - "Processing 43 of 1000 on page 1 - (1000 per page)\n", - "Processing 44 of 1000 on page 1 - (1000 per page)\n", - "Processing 45 of 1000 on page 1 - (1000 per page)\n", - "Processing 46 of 1000 on page 1 - (1000 per page)\n", - "Processing 47 of 1000 on page 1 - (1000 per page)\n", - "Processing 48 of 1000 on page 1 - (1000 per page)\n", - "Processing 49 of 1000 on page 1 - (1000 per page)\n", - "Processing 50 of 1000 on page 1 - (1000 per page)\n", - "Processing 51 of 1000 on page 1 - (1000 per page)\n", - "Processing 52 of 1000 on page 1 - (1000 per page)\n", - "Processing 53 of 1000 on page 1 - (1000 per page)\n", - "Processing 54 of 1000 on page 1 - (1000 per page)\n", - "Processing 55 of 1000 on page 1 - (1000 per page)\n", - "Processing 56 of 1000 on page 1 - (1000 per page)\n", - "Processing 57 of 1000 on page 1 - (1000 per page)\n", - "Processing 58 of 1000 on page 1 - (1000 per page)\n", - "Processing 59 of 1000 on page 1 - (1000 per page)\n", - "Processing 60 of 1000 on page 1 - (1000 per page)\n", - "Processing 61 of 1000 on page 1 - (1000 per page)\n", - "Processing 62 of 1000 on page 1 - (1000 per page)\n", - "Processing 63 of 1000 on page 1 - (1000 per page)\n", - "Processing 64 of 1000 on page 1 - (1000 per page)\n", - "Processing 65 of 1000 on page 1 - (1000 per page)\n", - "Processing 66 of 1000 on page 1 - (1000 per page)\n", - "Processing 67 of 1000 on page 1 - (1000 per page)\n", - "Processing 68 of 1000 on page 1 - (1000 per page)\n", - "Processing 69 of 1000 on page 1 - (1000 per page)\n", - "Processing 70 of 1000 on page 1 - (1000 per page)\n", - "Processing 71 of 1000 on page 1 - (1000 per page)\n", - "Processing 72 of 1000 on page 1 - (1000 per page)\n", - "Processing 73 of 1000 on page 1 - (1000 per page)\n", - "Processing 74 of 1000 on page 1 - (1000 per page)\n", - "Processing 75 of 1000 on page 1 - (1000 per page)\n", - "Processing 76 of 1000 on page 1 - (1000 per page)\n", - "Processing 77 of 1000 on page 1 - (1000 per page)\n", - "Processing 78 of 1000 on page 1 - (1000 per page)\n", - "Processing 79 of 1000 on page 1 - (1000 per page)\n", - "Processing 80 of 1000 on page 1 - (1000 per page)\n", - "Processing 81 of 1000 on page 1 - (1000 per page)\n", - "Processing 82 of 1000 on page 1 - (1000 per page)\n", - "Processing 83 of 1000 on page 1 - (1000 per page)\n", - "Processing 84 of 1000 on page 1 - (1000 per page)\n", - "Processing 85 of 1000 on page 1 - (1000 per page)\n", - "Processing 86 of 1000 on page 1 - (1000 per page)\n", - "Processing 87 of 1000 on page 1 - (1000 per page)\n", - "Processing 88 of 1000 on page 1 - (1000 per page)\n", - "Processing 89 of 1000 on page 1 - (1000 per page)\n", - "Processing 90 of 1000 on page 1 - (1000 per page)\n", - "Processing 91 of 1000 on page 1 - (1000 per page)\n", - "Processing 92 of 1000 on page 1 - (1000 per page)\n", - "Processing 93 of 1000 on page 1 - (1000 per page)\n", - "Processing 94 of 1000 on page 1 - (1000 per page)\n", - "Processing 95 of 1000 on page 1 - (1000 per page)\n", - "Processing 96 of 1000 on page 1 - (1000 per page)\n", - "Processing 97 of 1000 on page 1 - (1000 per page)\n", - "Processing 98 of 1000 on page 1 - (1000 per page)\n", - "Processing 99 of 1000 on page 1 - (1000 per page)\n", - "Processing 100 of 1000 on page 1 - (1000 per page)\n", - "Processing 101 of 1000 on page 1 - (1000 per page)\n", - "Processing 102 of 1000 on page 1 - (1000 per page)\n", - "Processing 103 of 1000 on page 1 - (1000 per page)\n", - "Processing 104 of 1000 on page 1 - (1000 per page)\n", - "Processing 105 of 1000 on page 1 - (1000 per page)\n", - "Processing 106 of 1000 on page 1 - (1000 per page)\n", - "Processing 107 of 1000 on page 1 - (1000 per page)\n", - "Processing 108 of 1000 on page 1 - (1000 per page)\n", - "Processing 109 of 1000 on page 1 - (1000 per page)\n", - "Processing 110 of 1000 on page 1 - (1000 per page)\n", - "Processing 111 of 1000 on page 1 - (1000 per page)\n", - "Processing 112 of 1000 on page 1 - (1000 per page)\n", - "Processing 113 of 1000 on page 1 - (1000 per page)\n", - "Processing 114 of 1000 on page 1 - (1000 per page)\n", - "Processing 115 of 1000 on page 1 - (1000 per page)\n", - "Processing 116 of 1000 on page 1 - (1000 per page)\n", - "Processing 117 of 1000 on page 1 - (1000 per page)\n", - "Processing 118 of 1000 on page 1 - (1000 per page)\n", - "Processing 119 of 1000 on page 1 - (1000 per page)\n", - "Processing 120 of 1000 on page 1 - (1000 per page)\n", - "Processing 121 of 1000 on page 1 - (1000 per page)\n", - "Processing 122 of 1000 on page 1 - (1000 per page)\n", - "Processing 123 of 1000 on page 1 - (1000 per page)\n", - "Processing 124 of 1000 on page 1 - (1000 per page)\n", - "Processing 125 of 1000 on page 1 - (1000 per page)\n", - "Processing 126 of 1000 on page 1 - (1000 per page)\n", - "Processing 127 of 1000 on page 1 - (1000 per page)\n", - "Processing 128 of 1000 on page 1 - (1000 per page)\n", - "Processing 129 of 1000 on page 1 - (1000 per page)\n", - "Processing 130 of 1000 on page 1 - (1000 per page)\n", - "Processing 131 of 1000 on page 1 - (1000 per page)\n", - "Processing 132 of 1000 on page 1 - (1000 per page)\n", - "Processing 133 of 1000 on page 1 - (1000 per page)\n", - "Processing 134 of 1000 on page 1 - (1000 per page)\n", - "Processing 135 of 1000 on page 1 - (1000 per page)\n", - "Processing 136 of 1000 on page 1 - (1000 per page)\n", - "Processing 137 of 1000 on page 1 - (1000 per page)\n", - "Processing 138 of 1000 on page 1 - (1000 per page)\n", - "Processing 139 of 1000 on page 1 - (1000 per page)\n", - "Processing 140 of 1000 on page 1 - (1000 per page)\n", - "Processing 141 of 1000 on page 1 - (1000 per page)\n", - "Processing 142 of 1000 on page 1 - (1000 per page)\n", - "Processing 143 of 1000 on page 1 - (1000 per page)\n", - "Processing 144 of 1000 on page 1 - (1000 per page)\n", - "Processing 145 of 1000 on page 1 - (1000 per page)\n", - "Processing 146 of 1000 on page 1 - (1000 per page)\n", - "Processing 147 of 1000 on page 1 - (1000 per page)\n", - "Processing 148 of 1000 on page 1 - (1000 per page)\n", - "Processing 149 of 1000 on page 1 - (1000 per page)\n", - "Processing 150 of 1000 on page 1 - (1000 per page)\n", - "Processing 151 of 1000 on page 1 - (1000 per page)\n", - "Processing 152 of 1000 on page 1 - (1000 per page)\n", - "Processing 153 of 1000 on page 1 - (1000 per page)\n", - "Processing 154 of 1000 on page 1 - (1000 per page)\n", - "Processing 155 of 1000 on page 1 - (1000 per page)\n", - "Processing 156 of 1000 on page 1 - (1000 per page)\n", - "Processing 157 of 1000 on page 1 - (1000 per page)\n", - "Processing 158 of 1000 on page 1 - (1000 per page)\n", - "Processing 159 of 1000 on page 1 - (1000 per page)\n", - "Processing 160 of 1000 on page 1 - (1000 per page)\n", - "Processing 161 of 1000 on page 1 - (1000 per page)\n", - "Processing 162 of 1000 on page 1 - (1000 per page)\n", - "Processing 163 of 1000 on page 1 - (1000 per page)\n", - "Processing 164 of 1000 on page 1 - (1000 per page)\n", - "Processing 165 of 1000 on page 1 - (1000 per page)\n", - "Processing 166 of 1000 on page 1 - (1000 per page)\n", - "Processing 167 of 1000 on page 1 - (1000 per page)\n", - "Processing 168 of 1000 on page 1 - (1000 per page)\n", - "Processing 169 of 1000 on page 1 - (1000 per page)\n", - "Processing 170 of 1000 on page 1 - (1000 per page)\n", - "Processing 171 of 1000 on page 1 - (1000 per page)\n", - "Processing 172 of 1000 on page 1 - (1000 per page)\n", - "Processing 173 of 1000 on page 1 - (1000 per page)\n", - "Processing 174 of 1000 on page 1 - (1000 per page)\n", - "Processing 175 of 1000 on page 1 - (1000 per page)\n", - "Processing 176 of 1000 on page 1 - (1000 per page)\n", - "Processing 177 of 1000 on page 1 - (1000 per page)\n", - "Processing 178 of 1000 on page 1 - (1000 per page)\n", - "Processing 179 of 1000 on page 1 - (1000 per page)\n", - "Processing 180 of 1000 on page 1 - (1000 per page)\n", - "Processing 181 of 1000 on page 1 - (1000 per page)\n", - "Processing 182 of 1000 on page 1 - (1000 per page)\n", - "Processing 183 of 1000 on page 1 - (1000 per page)\n", - "Processing 184 of 1000 on page 1 - (1000 per page)\n", - "Processing 185 of 1000 on page 1 - (1000 per page)\n", - "Processing 186 of 1000 on page 1 - (1000 per page)\n", - "Processing 187 of 1000 on page 1 - (1000 per page)\n", - "Processing 188 of 1000 on page 1 - (1000 per page)\n", - "Processing 189 of 1000 on page 1 - (1000 per page)\n", - "Processing 190 of 1000 on page 1 - (1000 per page)\n", - "Processing 191 of 1000 on page 1 - (1000 per page)\n", - "Processing 192 of 1000 on page 1 - (1000 per page)\n", - "Processing 193 of 1000 on page 1 - (1000 per page)\n", - "Processing 194 of 1000 on page 1 - (1000 per page)\n", - "Processing 195 of 1000 on page 1 - (1000 per page)\n", - "Processing 196 of 1000 on page 1 - (1000 per page)\n", - "Processing 197 of 1000 on page 1 - (1000 per page)\n", - "Processing 198 of 1000 on page 1 - (1000 per page)\n", - "Processing 199 of 1000 on page 1 - (1000 per page)\n", - "Processing 200 of 1000 on page 1 - (1000 per page)\n", - "Processing 201 of 1000 on page 1 - (1000 per page)\n", - "Processing 202 of 1000 on page 1 - (1000 per page)\n", - "Processing 203 of 1000 on page 1 - (1000 per page)\n", - "Processing 204 of 1000 on page 1 - (1000 per page)\n", - "Processing 205 of 1000 on page 1 - (1000 per page)\n", - "Processing 206 of 1000 on page 1 - (1000 per page)\n", - "Processing 207 of 1000 on page 1 - (1000 per page)\n", - "Processing 208 of 1000 on page 1 - (1000 per page)\n", - "Processing 209 of 1000 on page 1 - (1000 per page)\n", - "Processing 210 of 1000 on page 1 - (1000 per page)\n", - "Processing 211 of 1000 on page 1 - (1000 per page)\n", - "Processing 212 of 1000 on page 1 - (1000 per page)\n", - "Processing 213 of 1000 on page 1 - (1000 per page)\n", - "Processing 214 of 1000 on page 1 - (1000 per page)\n", - "Processing 215 of 1000 on page 1 - (1000 per page)\n", - "Processing 216 of 1000 on page 1 - (1000 per page)\n", - "Processing 217 of 1000 on page 1 - (1000 per page)\n", - "Processing 218 of 1000 on page 1 - (1000 per page)\n", - "Processing 219 of 1000 on page 1 - (1000 per page)\n", - "Processing 220 of 1000 on page 1 - (1000 per page)\n", - "Processing 221 of 1000 on page 1 - (1000 per page)\n", - "Processing 222 of 1000 on page 1 - (1000 per page)\n", - "Processing 223 of 1000 on page 1 - (1000 per page)\n", - "Processing 224 of 1000 on page 1 - (1000 per page)\n", - "Processing 225 of 1000 on page 1 - (1000 per page)\n", - "Processing 226 of 1000 on page 1 - (1000 per page)\n", - "Processing 227 of 1000 on page 1 - (1000 per page)\n", - "Processing 228 of 1000 on page 1 - (1000 per page)\n", - "Processing 229 of 1000 on page 1 - (1000 per page)\n", - "Processing 230 of 1000 on page 1 - (1000 per page)\n", - "Processing 231 of 1000 on page 1 - (1000 per page)\n", - "Processing 232 of 1000 on page 1 - (1000 per page)\n", - "Processing 233 of 1000 on page 1 - (1000 per page)\n", - "Processing 234 of 1000 on page 1 - (1000 per page)\n", - "Processing 235 of 1000 on page 1 - (1000 per page)\n", - "Processing 236 of 1000 on page 1 - (1000 per page)\n", - "Processing 237 of 1000 on page 1 - (1000 per page)\n", - "Processing 238 of 1000 on page 1 - (1000 per page)\n", - "Processing 239 of 1000 on page 1 - (1000 per page)\n", - "Processing 240 of 1000 on page 1 - (1000 per page)\n", - "Processing 241 of 1000 on page 1 - (1000 per page)\n", - "Processing 242 of 1000 on page 1 - (1000 per page)\n", - "Processing 243 of 1000 on page 1 - (1000 per page)\n", - "Processing 244 of 1000 on page 1 - (1000 per page)\n", - "Processing 245 of 1000 on page 1 - (1000 per page)\n", - "Processing 246 of 1000 on page 1 - (1000 per page)\n", - "Processing 247 of 1000 on page 1 - (1000 per page)\n", - "Processing 248 of 1000 on page 1 - (1000 per page)\n", - "Processing 249 of 1000 on page 1 - (1000 per page)\n", - "Processing 250 of 1000 on page 1 - (1000 per page)\n", - "Processing 251 of 1000 on page 1 - (1000 per page)\n", - "Processing 252 of 1000 on page 1 - (1000 per page)\n", - "Processing 253 of 1000 on page 1 - (1000 per page)\n", - "Processing 254 of 1000 on page 1 - (1000 per page)\n", - "Processing 255 of 1000 on page 1 - (1000 per page)\n", - "Processing 256 of 1000 on page 1 - (1000 per page)\n", - "Processing 257 of 1000 on page 1 - (1000 per page)\n", - "Processing 258 of 1000 on page 1 - (1000 per page)\n", - "Processing 259 of 1000 on page 1 - (1000 per page)\n", - "Processing 260 of 1000 on page 1 - (1000 per page)\n", - "Processing 261 of 1000 on page 1 - (1000 per page)\n", - "Processing 262 of 1000 on page 1 - (1000 per page)\n", - "Processing 263 of 1000 on page 1 - (1000 per page)\n", - "Processing 264 of 1000 on page 1 - (1000 per page)\n", - "Processing 265 of 1000 on page 1 - (1000 per page)\n", - "Processing 266 of 1000 on page 1 - (1000 per page)\n", - "Processing 267 of 1000 on page 1 - (1000 per page)\n", - "Processing 268 of 1000 on page 1 - (1000 per page)\n", - "Processing 269 of 1000 on page 1 - (1000 per page)\n", - "Processing 270 of 1000 on page 1 - (1000 per page)\n", - "Processing 271 of 1000 on page 1 - (1000 per page)\n", - "Processing 272 of 1000 on page 1 - (1000 per page)\n", - "Processing 273 of 1000 on page 1 - (1000 per page)\n", - "Processing 274 of 1000 on page 1 - (1000 per page)\n", - "Processing 275 of 1000 on page 1 - (1000 per page)\n", - "Processing 276 of 1000 on page 1 - (1000 per page)\n", - "Processing 277 of 1000 on page 1 - (1000 per page)\n", - "Processing 278 of 1000 on page 1 - (1000 per page)\n", - "Processing 279 of 1000 on page 1 - (1000 per page)\n", - "Processing 280 of 1000 on page 1 - (1000 per page)\n", - "Processing 281 of 1000 on page 1 - (1000 per page)\n", - "Processing 282 of 1000 on page 1 - (1000 per page)\n", - "Processing 283 of 1000 on page 1 - (1000 per page)\n", - "Processing 284 of 1000 on page 1 - (1000 per page)\n", - "Processing 285 of 1000 on page 1 - (1000 per page)\n", - "Processing 286 of 1000 on page 1 - (1000 per page)\n", - "Processing 287 of 1000 on page 1 - (1000 per page)\n", - "Processing 288 of 1000 on page 1 - (1000 per page)\n", - "Processing 289 of 1000 on page 1 - (1000 per page)\n", - "Processing 290 of 1000 on page 1 - (1000 per page)\n", - "Processing 291 of 1000 on page 1 - (1000 per page)\n", - "Processing 292 of 1000 on page 1 - (1000 per page)\n", - "Processing 293 of 1000 on page 1 - (1000 per page)\n", - "Processing 294 of 1000 on page 1 - (1000 per page)\n", - "Processing 295 of 1000 on page 1 - (1000 per page)\n", - "Processing 296 of 1000 on page 1 - (1000 per page)\n", - "Processing 297 of 1000 on page 1 - (1000 per page)\n", - "Processing 298 of 1000 on page 1 - (1000 per page)\n", - "Processing 299 of 1000 on page 1 - (1000 per page)\n", - "Processing 300 of 1000 on page 1 - (1000 per page)\n", - "Processing 301 of 1000 on page 1 - (1000 per page)\n", - "Processing 302 of 1000 on page 1 - (1000 per page)\n", - "Processing 303 of 1000 on page 1 - (1000 per page)\n", - "Processing 304 of 1000 on page 1 - (1000 per page)\n", - "Processing 305 of 1000 on page 1 - (1000 per page)\n", - "Processing 306 of 1000 on page 1 - (1000 per page)\n", - "Processing 307 of 1000 on page 1 - (1000 per page)\n", - "Processing 308 of 1000 on page 1 - (1000 per page)\n", - "Processing 309 of 1000 on page 1 - (1000 per page)\n", - "Processing 310 of 1000 on page 1 - (1000 per page)\n", - "Processing 311 of 1000 on page 1 - (1000 per page)\n", - "Processing 312 of 1000 on page 1 - (1000 per page)\n", - "Processing 313 of 1000 on page 1 - (1000 per page)\n", - "Processing 314 of 1000 on page 1 - (1000 per page)\n", - "Processing 315 of 1000 on page 1 - (1000 per page)\n", - "Processing 316 of 1000 on page 1 - (1000 per page)\n", - "Processing 317 of 1000 on page 1 - (1000 per page)\n", - "Processing 318 of 1000 on page 1 - (1000 per page)\n", - "Processing 319 of 1000 on page 1 - (1000 per page)\n", - "Processing 320 of 1000 on page 1 - (1000 per page)\n", - "Processing 321 of 1000 on page 1 - (1000 per page)\n", - "Processing 322 of 1000 on page 1 - (1000 per page)\n", - "Processing 323 of 1000 on page 1 - (1000 per page)\n", - "Processing 324 of 1000 on page 1 - (1000 per page)\n", - "Processing 325 of 1000 on page 1 - (1000 per page)\n", - "Processing 326 of 1000 on page 1 - (1000 per page)\n", - "Processing 327 of 1000 on page 1 - (1000 per page)\n", - "Processing 328 of 1000 on page 1 - (1000 per page)\n", - "Processing 329 of 1000 on page 1 - (1000 per page)\n", - "Processing 330 of 1000 on page 1 - (1000 per page)\n", - "Processing 331 of 1000 on page 1 - (1000 per page)\n", - "Processing 332 of 1000 on page 1 - (1000 per page)\n", - "Processing 333 of 1000 on page 1 - (1000 per page)\n", - "Processing 334 of 1000 on page 1 - (1000 per page)\n", - "Processing 335 of 1000 on page 1 - (1000 per page)\n", - "Processing 336 of 1000 on page 1 - (1000 per page)\n", - "Processing 337 of 1000 on page 1 - (1000 per page)\n", - "Processing 338 of 1000 on page 1 - (1000 per page)\n", - "Processing 339 of 1000 on page 1 - (1000 per page)\n", - "Processing 340 of 1000 on page 1 - (1000 per page)\n", - "Processing 341 of 1000 on page 1 - (1000 per page)\n", - "Processing 342 of 1000 on page 1 - (1000 per page)\n", - "Processing 343 of 1000 on page 1 - (1000 per page)\n", - "Processing 344 of 1000 on page 1 - (1000 per page)\n", - "Processing 345 of 1000 on page 1 - (1000 per page)\n", - "Processing 346 of 1000 on page 1 - (1000 per page)\n", - "Processing 347 of 1000 on page 1 - (1000 per page)\n", - "Processing 348 of 1000 on page 1 - (1000 per page)\n", - "Processing 349 of 1000 on page 1 - (1000 per page)\n", - "Processing 350 of 1000 on page 1 - (1000 per page)\n", - "Processing 351 of 1000 on page 1 - (1000 per page)\n", - "Processing 352 of 1000 on page 1 - (1000 per page)\n", - "Processing 353 of 1000 on page 1 - (1000 per page)\n", - "Processing 354 of 1000 on page 1 - (1000 per page)\n", - "Processing 355 of 1000 on page 1 - (1000 per page)\n", - "Processing 356 of 1000 on page 1 - (1000 per page)\n", - "Processing 357 of 1000 on page 1 - (1000 per page)\n", - "Processing 358 of 1000 on page 1 - (1000 per page)\n", - "Processing 359 of 1000 on page 1 - (1000 per page)\n", - "Processing 360 of 1000 on page 1 - (1000 per page)\n", - "Processing 361 of 1000 on page 1 - (1000 per page)\n", - "Processing 362 of 1000 on page 1 - (1000 per page)\n", - "Processing 363 of 1000 on page 1 - (1000 per page)\n", - "Processing 364 of 1000 on page 1 - (1000 per page)\n", - "Processing 365 of 1000 on page 1 - (1000 per page)\n", - "Processing 366 of 1000 on page 1 - (1000 per page)\n", - "Processing 367 of 1000 on page 1 - (1000 per page)\n", - "Processing 368 of 1000 on page 1 - (1000 per page)\n", - "Processing 369 of 1000 on page 1 - (1000 per page)\n", - "Processing 370 of 1000 on page 1 - (1000 per page)\n", - "Processing 371 of 1000 on page 1 - (1000 per page)\n", - "Processing 372 of 1000 on page 1 - (1000 per page)\n", - "Processing 373 of 1000 on page 1 - (1000 per page)\n", - "Processing 374 of 1000 on page 1 - (1000 per page)\n", - "Processing 375 of 1000 on page 1 - (1000 per page)\n", - "Processing 376 of 1000 on page 1 - (1000 per page)\n", - "Processing 377 of 1000 on page 1 - (1000 per page)\n", - "Processing 378 of 1000 on page 1 - (1000 per page)\n", - "Processing 379 of 1000 on page 1 - (1000 per page)\n", - "Processing 380 of 1000 on page 1 - (1000 per page)\n", - "Processing 381 of 1000 on page 1 - (1000 per page)\n", - "Processing 382 of 1000 on page 1 - (1000 per page)\n", - "Processing 383 of 1000 on page 1 - (1000 per page)\n", - "Processing 384 of 1000 on page 1 - (1000 per page)\n", - "Processing 385 of 1000 on page 1 - (1000 per page)\n", - "Processing 386 of 1000 on page 1 - (1000 per page)\n", - "Processing 387 of 1000 on page 1 - (1000 per page)\n", - "Processing 388 of 1000 on page 1 - (1000 per page)\n", - "Processing 389 of 1000 on page 1 - (1000 per page)\n", - "Processing 390 of 1000 on page 1 - (1000 per page)\n", - "Processing 391 of 1000 on page 1 - (1000 per page)\n", - "Processing 392 of 1000 on page 1 - (1000 per page)\n", - "Processing 393 of 1000 on page 1 - (1000 per page)\n", - "Processing 394 of 1000 on page 1 - (1000 per page)\n", - "Processing 395 of 1000 on page 1 - (1000 per page)\n", - "Processing 396 of 1000 on page 1 - (1000 per page)\n", - "Processing 397 of 1000 on page 1 - (1000 per page)\n", - "Processing 398 of 1000 on page 1 - (1000 per page)\n", - "Processing 399 of 1000 on page 1 - (1000 per page)\n", - "Processing 400 of 1000 on page 1 - (1000 per page)\n", - "Processing 401 of 1000 on page 1 - (1000 per page)\n", - "Processing 402 of 1000 on page 1 - (1000 per page)\n", - "Processing 403 of 1000 on page 1 - (1000 per page)\n", - "Processing 404 of 1000 on page 1 - (1000 per page)\n", - "Processing 405 of 1000 on page 1 - (1000 per page)\n", - "Processing 406 of 1000 on page 1 - (1000 per page)\n", - "Processing 407 of 1000 on page 1 - (1000 per page)\n", - "Processing 408 of 1000 on page 1 - (1000 per page)\n", - "Processing 409 of 1000 on page 1 - (1000 per page)\n", - "Processing 410 of 1000 on page 1 - (1000 per page)\n", - "Processing 411 of 1000 on page 1 - (1000 per page)\n", - "Processing 412 of 1000 on page 1 - (1000 per page)\n", - "Processing 413 of 1000 on page 1 - (1000 per page)\n", - "Processing 414 of 1000 on page 1 - (1000 per page)\n", - "Processing 415 of 1000 on page 1 - (1000 per page)\n", - "Processing 416 of 1000 on page 1 - (1000 per page)\n", - "Processing 417 of 1000 on page 1 - (1000 per page)\n", - "Processing 418 of 1000 on page 1 - (1000 per page)\n", - "Processing 419 of 1000 on page 1 - (1000 per page)\n", - "Processing 420 of 1000 on page 1 - (1000 per page)\n", - "Processing 421 of 1000 on page 1 - (1000 per page)\n", - "Processing 422 of 1000 on page 1 - (1000 per page)\n", - "Processing 423 of 1000 on page 1 - (1000 per page)\n", - "Processing 424 of 1000 on page 1 - (1000 per page)\n", - "Processing 425 of 1000 on page 1 - (1000 per page)\n", - "Processing 426 of 1000 on page 1 - (1000 per page)\n", - "Processing 427 of 1000 on page 1 - (1000 per page)\n", - "Processing 428 of 1000 on page 1 - (1000 per page)\n", - "Processing 429 of 1000 on page 1 - (1000 per page)\n", - "Processing 430 of 1000 on page 1 - (1000 per page)\n", - "Processing 431 of 1000 on page 1 - (1000 per page)\n", - "Processing 432 of 1000 on page 1 - (1000 per page)\n", - "Processing 433 of 1000 on page 1 - (1000 per page)\n", - "Processing 434 of 1000 on page 1 - (1000 per page)\n", - "Processing 435 of 1000 on page 1 - (1000 per page)\n", - "Processing 436 of 1000 on page 1 - (1000 per page)\n", - "Processing 437 of 1000 on page 1 - (1000 per page)\n", - "Processing 438 of 1000 on page 1 - (1000 per page)\n", - "Processing 439 of 1000 on page 1 - (1000 per page)\n", - "Processing 440 of 1000 on page 1 - (1000 per page)\n", - "Processing 441 of 1000 on page 1 - (1000 per page)\n", - "Processing 442 of 1000 on page 1 - (1000 per page)\n", - "Processing 443 of 1000 on page 1 - (1000 per page)\n", - "Processing 444 of 1000 on page 1 - (1000 per page)\n", - "Processing 445 of 1000 on page 1 - (1000 per page)\n", - "Processing 446 of 1000 on page 1 - (1000 per page)\n", - "Processing 447 of 1000 on page 1 - (1000 per page)\n", - "Processing 448 of 1000 on page 1 - (1000 per page)\n", - "Processing 449 of 1000 on page 1 - (1000 per page)\n", - "Processing 450 of 1000 on page 1 - (1000 per page)\n", - "Processing 451 of 1000 on page 1 - (1000 per page)\n", - "Processing 452 of 1000 on page 1 - (1000 per page)\n", - "Processing 453 of 1000 on page 1 - (1000 per page)\n", - "Processing 454 of 1000 on page 1 - (1000 per page)\n", - "Processing 455 of 1000 on page 1 - (1000 per page)\n", - "Processing 456 of 1000 on page 1 - (1000 per page)\n", - "Processing 457 of 1000 on page 1 - (1000 per page)\n", - "Processing 458 of 1000 on page 1 - (1000 per page)\n", - "Processing 459 of 1000 on page 1 - (1000 per page)\n", - "Processing 460 of 1000 on page 1 - (1000 per page)\n", - "Processing 461 of 1000 on page 1 - (1000 per page)\n", - "Processing 462 of 1000 on page 1 - (1000 per page)\n", - "Processing 463 of 1000 on page 1 - (1000 per page)\n", - "Processing 464 of 1000 on page 1 - (1000 per page)\n", - "Processing 465 of 1000 on page 1 - (1000 per page)\n", - "Processing 466 of 1000 on page 1 - (1000 per page)\n", - "Processing 467 of 1000 on page 1 - (1000 per page)\n", - "Processing 468 of 1000 on page 1 - (1000 per page)\n", - "Processing 469 of 1000 on page 1 - (1000 per page)\n", - "Processing 470 of 1000 on page 1 - (1000 per page)\n", - "Processing 471 of 1000 on page 1 - (1000 per page)\n", - "Processing 472 of 1000 on page 1 - (1000 per page)\n", - "Processing 473 of 1000 on page 1 - (1000 per page)\n", - "Processing 474 of 1000 on page 1 - (1000 per page)\n", - "Processing 475 of 1000 on page 1 - (1000 per page)\n", - "Processing 476 of 1000 on page 1 - (1000 per page)\n", - "Processing 477 of 1000 on page 1 - (1000 per page)\n", - "Processing 478 of 1000 on page 1 - (1000 per page)\n", - "Processing 479 of 1000 on page 1 - (1000 per page)\n", - "Processing 480 of 1000 on page 1 - (1000 per page)\n", - "Processing 481 of 1000 on page 1 - (1000 per page)\n", - "Processing 482 of 1000 on page 1 - (1000 per page)\n", - "Processing 483 of 1000 on page 1 - (1000 per page)\n", - "Processing 484 of 1000 on page 1 - (1000 per page)\n", - "Processing 485 of 1000 on page 1 - (1000 per page)\n", - "Processing 486 of 1000 on page 1 - (1000 per page)\n", - "Processing 487 of 1000 on page 1 - (1000 per page)\n", - "Processing 488 of 1000 on page 1 - (1000 per page)\n", - "Processing 489 of 1000 on page 1 - (1000 per page)\n", - "Processing 490 of 1000 on page 1 - (1000 per page)\n", - "Processing 491 of 1000 on page 1 - (1000 per page)\n", - "Processing 492 of 1000 on page 1 - (1000 per page)\n", - "Processing 493 of 1000 on page 1 - (1000 per page)\n", - "Processing 494 of 1000 on page 1 - (1000 per page)\n", - "Processing 495 of 1000 on page 1 - (1000 per page)\n", - "Processing 496 of 1000 on page 1 - (1000 per page)\n", - "Processing 497 of 1000 on page 1 - (1000 per page)\n", - "Processing 498 of 1000 on page 1 - (1000 per page)\n", - "Processing 499 of 1000 on page 1 - (1000 per page)\n", - "Processing 500 of 1000 on page 1 - (1000 per page)\n", - "Processing 501 of 1000 on page 1 - (1000 per page)\n", - "Processing 502 of 1000 on page 1 - (1000 per page)\n", - "Processing 503 of 1000 on page 1 - (1000 per page)\n", - "Processing 504 of 1000 on page 1 - (1000 per page)\n", - "Processing 505 of 1000 on page 1 - (1000 per page)\n", - "Processing 506 of 1000 on page 1 - (1000 per page)\n", - "Processing 507 of 1000 on page 1 - (1000 per page)\n", - "Processing 508 of 1000 on page 1 - (1000 per page)\n", - "Processing 509 of 1000 on page 1 - (1000 per page)\n", - "Processing 510 of 1000 on page 1 - (1000 per page)\n", - "Processing 511 of 1000 on page 1 - (1000 per page)\n", - "Processing 512 of 1000 on page 1 - (1000 per page)\n", - "Processing 513 of 1000 on page 1 - (1000 per page)\n", - "Processing 514 of 1000 on page 1 - (1000 per page)\n", - "Processing 515 of 1000 on page 1 - (1000 per page)\n", - "Processing 516 of 1000 on page 1 - (1000 per page)\n", - "Processing 517 of 1000 on page 1 - (1000 per page)\n", - "Processing 518 of 1000 on page 1 - (1000 per page)\n", - "Processing 519 of 1000 on page 1 - (1000 per page)\n", - "Processing 520 of 1000 on page 1 - (1000 per page)\n", - "Processing 521 of 1000 on page 1 - (1000 per page)\n", - "Processing 522 of 1000 on page 1 - (1000 per page)\n", - "Processing 523 of 1000 on page 1 - (1000 per page)\n", - "Processing 524 of 1000 on page 1 - (1000 per page)\n", - "Processing 525 of 1000 on page 1 - (1000 per page)\n", - "Processing 526 of 1000 on page 1 - (1000 per page)\n", - "Processing 527 of 1000 on page 1 - (1000 per page)\n", - "Processing 528 of 1000 on page 1 - (1000 per page)\n", - "Processing 529 of 1000 on page 1 - (1000 per page)\n", - "Processing 530 of 1000 on page 1 - (1000 per page)\n", - "Processing 531 of 1000 on page 1 - (1000 per page)\n", - "Processing 532 of 1000 on page 1 - (1000 per page)\n", - "Processing 533 of 1000 on page 1 - (1000 per page)\n", - "Processing 534 of 1000 on page 1 - (1000 per page)\n", - "Processing 535 of 1000 on page 1 - (1000 per page)\n", - "Processing 536 of 1000 on page 1 - (1000 per page)\n", - "Processing 537 of 1000 on page 1 - (1000 per page)\n", - "Processing 538 of 1000 on page 1 - (1000 per page)\n", - "Processing 539 of 1000 on page 1 - (1000 per page)\n", - "Processing 540 of 1000 on page 1 - (1000 per page)\n", - "Processing 541 of 1000 on page 1 - (1000 per page)\n", - "Processing 542 of 1000 on page 1 - (1000 per page)\n", - "Processing 543 of 1000 on page 1 - (1000 per page)\n", - "Processing 544 of 1000 on page 1 - (1000 per page)\n", - "Processing 545 of 1000 on page 1 - (1000 per page)\n", - "Processing 546 of 1000 on page 1 - (1000 per page)\n", - "Processing 547 of 1000 on page 1 - (1000 per page)\n", - "Processing 548 of 1000 on page 1 - (1000 per page)\n", - "Processing 549 of 1000 on page 1 - (1000 per page)\n", - "Processing 550 of 1000 on page 1 - (1000 per page)\n", - "Processing 551 of 1000 on page 1 - (1000 per page)\n", - "Processing 552 of 1000 on page 1 - (1000 per page)\n", - "Processing 553 of 1000 on page 1 - (1000 per page)\n", - "Processing 554 of 1000 on page 1 - (1000 per page)\n", - "Processing 555 of 1000 on page 1 - (1000 per page)\n", - "Processing 556 of 1000 on page 1 - (1000 per page)\n", - "Processing 557 of 1000 on page 1 - (1000 per page)\n", - "Processing 558 of 1000 on page 1 - (1000 per page)\n", - "Processing 559 of 1000 on page 1 - (1000 per page)\n", - "Processing 560 of 1000 on page 1 - (1000 per page)\n", - "Processing 561 of 1000 on page 1 - (1000 per page)\n", - "Processing 562 of 1000 on page 1 - (1000 per page)\n", - "Processing 563 of 1000 on page 1 - (1000 per page)\n", - "Processing 564 of 1000 on page 1 - (1000 per page)\n", - "Processing 565 of 1000 on page 1 - (1000 per page)\n", - "Processing 566 of 1000 on page 1 - (1000 per page)\n", - "Processing 567 of 1000 on page 1 - (1000 per page)\n", - "Processing 568 of 1000 on page 1 - (1000 per page)\n", - "Processing 569 of 1000 on page 1 - (1000 per page)\n", - "Processing 570 of 1000 on page 1 - (1000 per page)\n", - "Processing 571 of 1000 on page 1 - (1000 per page)\n", - "Processing 572 of 1000 on page 1 - (1000 per page)\n", - "Processing 573 of 1000 on page 1 - (1000 per page)\n", - "Processing 574 of 1000 on page 1 - (1000 per page)\n", - "Processing 575 of 1000 on page 1 - (1000 per page)\n", - "Processing 576 of 1000 on page 1 - (1000 per page)\n", - "Processing 577 of 1000 on page 1 - (1000 per page)\n", - "Processing 578 of 1000 on page 1 - (1000 per page)\n", - "Processing 579 of 1000 on page 1 - (1000 per page)\n", - "Processing 580 of 1000 on page 1 - (1000 per page)\n", - "Processing 581 of 1000 on page 1 - (1000 per page)\n", - "Processing 582 of 1000 on page 1 - (1000 per page)\n", - "Processing 583 of 1000 on page 1 - (1000 per page)\n", - "Processing 584 of 1000 on page 1 - (1000 per page)\n", - "Processing 585 of 1000 on page 1 - (1000 per page)\n", - "Processing 586 of 1000 on page 1 - (1000 per page)\n", - "Processing 587 of 1000 on page 1 - (1000 per page)\n", - "Processing 588 of 1000 on page 1 - (1000 per page)\n", - "Processing 589 of 1000 on page 1 - (1000 per page)\n", - "Processing 590 of 1000 on page 1 - (1000 per page)\n", - "Processing 591 of 1000 on page 1 - (1000 per page)\n", - "Processing 592 of 1000 on page 1 - (1000 per page)\n", - "Processing 593 of 1000 on page 1 - (1000 per page)\n", - "Processing 594 of 1000 on page 1 - (1000 per page)\n", - "Processing 595 of 1000 on page 1 - (1000 per page)\n", - "Processing 596 of 1000 on page 1 - (1000 per page)\n", - "Processing 597 of 1000 on page 1 - (1000 per page)\n", - "Processing 598 of 1000 on page 1 - (1000 per page)\n", - "Processing 599 of 1000 on page 1 - (1000 per page)\n", - "Processing 600 of 1000 on page 1 - (1000 per page)\n", - "Processing 601 of 1000 on page 1 - (1000 per page)\n", - "Processing 602 of 1000 on page 1 - (1000 per page)\n", - "Processing 603 of 1000 on page 1 - (1000 per page)\n", - "Processing 604 of 1000 on page 1 - (1000 per page)\n", - "Processing 605 of 1000 on page 1 - (1000 per page)\n", - "Processing 606 of 1000 on page 1 - (1000 per page)\n", - "Processing 607 of 1000 on page 1 - (1000 per page)\n", - "Processing 608 of 1000 on page 1 - (1000 per page)\n", - "Processing 609 of 1000 on page 1 - (1000 per page)\n", - "Processing 610 of 1000 on page 1 - (1000 per page)\n", - "Processing 611 of 1000 on page 1 - (1000 per page)\n", - "Processing 612 of 1000 on page 1 - (1000 per page)\n", - "Processing 613 of 1000 on page 1 - (1000 per page)\n", - "Processing 614 of 1000 on page 1 - (1000 per page)\n", - "Processing 615 of 1000 on page 1 - (1000 per page)\n", - "Processing 616 of 1000 on page 1 - (1000 per page)\n", - "Processing 617 of 1000 on page 1 - (1000 per page)\n", - "Processing 618 of 1000 on page 1 - (1000 per page)\n", - "Processing 619 of 1000 on page 1 - (1000 per page)\n", - "Processing 620 of 1000 on page 1 - (1000 per page)\n", - "Processing 621 of 1000 on page 1 - (1000 per page)\n", - "Processing 622 of 1000 on page 1 - (1000 per page)\n", - "Processing 623 of 1000 on page 1 - (1000 per page)\n", - "Processing 624 of 1000 on page 1 - (1000 per page)\n", - "Processing 625 of 1000 on page 1 - (1000 per page)\n", - "Processing 626 of 1000 on page 1 - (1000 per page)\n", - "Processing 627 of 1000 on page 1 - (1000 per page)\n", - "Processing 628 of 1000 on page 1 - (1000 per page)\n", - "Processing 629 of 1000 on page 1 - (1000 per page)\n", - "Processing 630 of 1000 on page 1 - (1000 per page)\n", - "Processing 631 of 1000 on page 1 - (1000 per page)\n", - "Processing 632 of 1000 on page 1 - (1000 per page)\n", - "Processing 633 of 1000 on page 1 - (1000 per page)\n", - "Processing 634 of 1000 on page 1 - (1000 per page)\n", - "Processing 635 of 1000 on page 1 - (1000 per page)\n", - "Processing 636 of 1000 on page 1 - (1000 per page)\n", - "Processing 637 of 1000 on page 1 - (1000 per page)\n", - "Processing 638 of 1000 on page 1 - (1000 per page)\n", - "Processing 639 of 1000 on page 1 - (1000 per page)\n", - "Processing 640 of 1000 on page 1 - (1000 per page)\n", - "Processing 641 of 1000 on page 1 - (1000 per page)\n", - "Processing 642 of 1000 on page 1 - (1000 per page)\n", - "Processing 643 of 1000 on page 1 - (1000 per page)\n", - "Processing 644 of 1000 on page 1 - (1000 per page)\n", - "Processing 645 of 1000 on page 1 - (1000 per page)\n", - "Processing 646 of 1000 on page 1 - (1000 per page)\n", - "Processing 647 of 1000 on page 1 - (1000 per page)\n", - "Processing 648 of 1000 on page 1 - (1000 per page)\n", - "Processing 649 of 1000 on page 1 - (1000 per page)\n", - "Processing 650 of 1000 on page 1 - (1000 per page)\n", - "Processing 651 of 1000 on page 1 - (1000 per page)\n", - "Processing 652 of 1000 on page 1 - (1000 per page)\n", - "Processing 653 of 1000 on page 1 - (1000 per page)\n", - "Processing 654 of 1000 on page 1 - (1000 per page)\n", - "Processing 655 of 1000 on page 1 - (1000 per page)\n", - "Processing 656 of 1000 on page 1 - (1000 per page)\n", - "Processing 657 of 1000 on page 1 - (1000 per page)\n", - "Processing 658 of 1000 on page 1 - (1000 per page)\n", - "Processing 659 of 1000 on page 1 - (1000 per page)\n", - "Processing 660 of 1000 on page 1 - (1000 per page)\n", - "Processing 661 of 1000 on page 1 - (1000 per page)\n", - "Processing 662 of 1000 on page 1 - (1000 per page)\n", - "Processing 663 of 1000 on page 1 - (1000 per page)\n", - "Processing 664 of 1000 on page 1 - (1000 per page)\n", - "Processing 665 of 1000 on page 1 - (1000 per page)\n", - "Processing 666 of 1000 on page 1 - (1000 per page)\n", - "Processing 667 of 1000 on page 1 - (1000 per page)\n", - "Processing 668 of 1000 on page 1 - (1000 per page)\n", - "Processing 669 of 1000 on page 1 - (1000 per page)\n", - "Processing 670 of 1000 on page 1 - (1000 per page)\n", - "Processing 671 of 1000 on page 1 - (1000 per page)\n", - "Processing 672 of 1000 on page 1 - (1000 per page)\n", - "Processing 673 of 1000 on page 1 - (1000 per page)\n", - "Processing 674 of 1000 on page 1 - (1000 per page)\n", - "Processing 675 of 1000 on page 1 - (1000 per page)\n", - "Processing 676 of 1000 on page 1 - (1000 per page)\n", - "Processing 677 of 1000 on page 1 - (1000 per page)\n", - "Processing 678 of 1000 on page 1 - (1000 per page)\n", - "Processing 679 of 1000 on page 1 - (1000 per page)\n", - "Processing 680 of 1000 on page 1 - (1000 per page)\n", - "Processing 681 of 1000 on page 1 - (1000 per page)\n", - "Processing 682 of 1000 on page 1 - (1000 per page)\n", - "Processing 683 of 1000 on page 1 - (1000 per page)\n", - "Processing 684 of 1000 on page 1 - (1000 per page)\n", - "Processing 685 of 1000 on page 1 - (1000 per page)\n", - "Processing 686 of 1000 on page 1 - (1000 per page)\n", - "Processing 687 of 1000 on page 1 - (1000 per page)\n", - "Processing 688 of 1000 on page 1 - (1000 per page)\n", - "Processing 689 of 1000 on page 1 - (1000 per page)\n", - "Processing 690 of 1000 on page 1 - (1000 per page)\n", - "Processing 691 of 1000 on page 1 - (1000 per page)\n", - "Processing 692 of 1000 on page 1 - (1000 per page)\n", - "Processing 693 of 1000 on page 1 - (1000 per page)\n", - "Processing 694 of 1000 on page 1 - (1000 per page)\n", - "Processing 695 of 1000 on page 1 - (1000 per page)\n", - "Processing 696 of 1000 on page 1 - (1000 per page)\n", - "Processing 697 of 1000 on page 1 - (1000 per page)\n", - "Processing 698 of 1000 on page 1 - (1000 per page)\n", - "Processing 699 of 1000 on page 1 - (1000 per page)\n", - "Processing 700 of 1000 on page 1 - (1000 per page)\n", - "Processing 701 of 1000 on page 1 - (1000 per page)\n", - "Processing 702 of 1000 on page 1 - (1000 per page)\n", - "Processing 703 of 1000 on page 1 - (1000 per page)\n", - "Processing 704 of 1000 on page 1 - (1000 per page)\n", - "Processing 705 of 1000 on page 1 - (1000 per page)\n", - "Processing 706 of 1000 on page 1 - (1000 per page)\n", - "Processing 707 of 1000 on page 1 - (1000 per page)\n", - "Processing 708 of 1000 on page 1 - (1000 per page)\n", - "Processing 709 of 1000 on page 1 - (1000 per page)\n", - "Processing 710 of 1000 on page 1 - (1000 per page)\n", - "Processing 711 of 1000 on page 1 - (1000 per page)\n", - "Processing 712 of 1000 on page 1 - (1000 per page)\n", - "Processing 713 of 1000 on page 1 - (1000 per page)\n", - "Processing 714 of 1000 on page 1 - (1000 per page)\n", - "Processing 715 of 1000 on page 1 - (1000 per page)\n", - "Processing 716 of 1000 on page 1 - (1000 per page)\n", - "Processing 717 of 1000 on page 1 - (1000 per page)\n", - "Processing 718 of 1000 on page 1 - (1000 per page)\n", - "Processing 719 of 1000 on page 1 - (1000 per page)\n", - "Processing 720 of 1000 on page 1 - (1000 per page)\n", - "Processing 721 of 1000 on page 1 - (1000 per page)\n", - "Processing 722 of 1000 on page 1 - (1000 per page)\n", - "Processing 723 of 1000 on page 1 - (1000 per page)\n", - "Processing 724 of 1000 on page 1 - (1000 per page)\n", - "Processing 725 of 1000 on page 1 - (1000 per page)\n", - "Processing 726 of 1000 on page 1 - (1000 per page)\n", - "Processing 727 of 1000 on page 1 - (1000 per page)\n", - "Processing 728 of 1000 on page 1 - (1000 per page)\n", - "Processing 729 of 1000 on page 1 - (1000 per page)\n", - "Processing 730 of 1000 on page 1 - (1000 per page)\n", - "Processing 731 of 1000 on page 1 - (1000 per page)\n", - "Processing 732 of 1000 on page 1 - (1000 per page)\n", - "Processing 733 of 1000 on page 1 - (1000 per page)\n", - "Processing 734 of 1000 on page 1 - (1000 per page)\n", - "Processing 735 of 1000 on page 1 - (1000 per page)\n", - "Processing 736 of 1000 on page 1 - (1000 per page)\n", - "Processing 737 of 1000 on page 1 - (1000 per page)\n", - "Processing 738 of 1000 on page 1 - (1000 per page)\n", - "Processing 739 of 1000 on page 1 - (1000 per page)\n", - "Processing 740 of 1000 on page 1 - (1000 per page)\n", - "Processing 741 of 1000 on page 1 - (1000 per page)\n", - "Processing 742 of 1000 on page 1 - (1000 per page)\n", - "Processing 743 of 1000 on page 1 - (1000 per page)\n", - "Processing 744 of 1000 on page 1 - (1000 per page)\n", - "Processing 745 of 1000 on page 1 - (1000 per page)\n", - "Processing 746 of 1000 on page 1 - (1000 per page)\n", - "Processing 747 of 1000 on page 1 - (1000 per page)\n", - "Processing 748 of 1000 on page 1 - (1000 per page)\n", - "Processing 749 of 1000 on page 1 - (1000 per page)\n", - "Processing 750 of 1000 on page 1 - (1000 per page)\n", - "Processing 751 of 1000 on page 1 - (1000 per page)\n", - "Processing 752 of 1000 on page 1 - (1000 per page)\n", - "Processing 753 of 1000 on page 1 - (1000 per page)\n", - "Processing 754 of 1000 on page 1 - (1000 per page)\n", - "Processing 755 of 1000 on page 1 - (1000 per page)\n", - "Processing 756 of 1000 on page 1 - (1000 per page)\n", - "Processing 757 of 1000 on page 1 - (1000 per page)\n", - "Processing 758 of 1000 on page 1 - (1000 per page)\n", - "Processing 759 of 1000 on page 1 - (1000 per page)\n", - "Processing 760 of 1000 on page 1 - (1000 per page)\n", - "Processing 761 of 1000 on page 1 - (1000 per page)\n", - "Processing 762 of 1000 on page 1 - (1000 per page)\n", - "Processing 763 of 1000 on page 1 - (1000 per page)\n", - "Processing 764 of 1000 on page 1 - (1000 per page)\n", - "Processing 765 of 1000 on page 1 - (1000 per page)\n", - "Processing 766 of 1000 on page 1 - (1000 per page)\n", - "Processing 767 of 1000 on page 1 - (1000 per page)\n", - "Processing 768 of 1000 on page 1 - (1000 per page)\n", - "Processing 769 of 1000 on page 1 - (1000 per page)\n", - "Processing 770 of 1000 on page 1 - (1000 per page)\n", - "Processing 771 of 1000 on page 1 - (1000 per page)\n", - "Processing 772 of 1000 on page 1 - (1000 per page)\n", - "Processing 773 of 1000 on page 1 - (1000 per page)\n", - "Processing 774 of 1000 on page 1 - (1000 per page)\n", - "Processing 775 of 1000 on page 1 - (1000 per page)\n", - "Processing 776 of 1000 on page 1 - (1000 per page)\n", - "Processing 777 of 1000 on page 1 - (1000 per page)\n", - "Processing 778 of 1000 on page 1 - (1000 per page)\n", - "Processing 779 of 1000 on page 1 - (1000 per page)\n", - "Processing 780 of 1000 on page 1 - (1000 per page)\n", - "Processing 781 of 1000 on page 1 - (1000 per page)\n", - "Processing 782 of 1000 on page 1 - (1000 per page)\n", - "Processing 783 of 1000 on page 1 - (1000 per page)\n", - "Processing 784 of 1000 on page 1 - (1000 per page)\n", - "Processing 785 of 1000 on page 1 - (1000 per page)\n", - "Processing 786 of 1000 on page 1 - (1000 per page)\n", - "Processing 787 of 1000 on page 1 - (1000 per page)\n", - "Processing 788 of 1000 on page 1 - (1000 per page)\n", - "Processing 789 of 1000 on page 1 - (1000 per page)\n", - "Processing 790 of 1000 on page 1 - (1000 per page)\n", - "Processing 791 of 1000 on page 1 - (1000 per page)\n", - "Processing 792 of 1000 on page 1 - (1000 per page)\n", - "Processing 793 of 1000 on page 1 - (1000 per page)\n", - "Processing 794 of 1000 on page 1 - (1000 per page)\n", - "Processing 795 of 1000 on page 1 - (1000 per page)\n", - "Processing 796 of 1000 on page 1 - (1000 per page)\n", - "Processing 797 of 1000 on page 1 - (1000 per page)\n", - "Processing 798 of 1000 on page 1 - (1000 per page)\n", - "Processing 799 of 1000 on page 1 - (1000 per page)\n", - "Processing 800 of 1000 on page 1 - (1000 per page)\n", - "Processing 801 of 1000 on page 1 - (1000 per page)\n", - "Processing 802 of 1000 on page 1 - (1000 per page)\n", - "Processing 803 of 1000 on page 1 - (1000 per page)\n", - "Processing 804 of 1000 on page 1 - (1000 per page)\n", - "Processing 805 of 1000 on page 1 - (1000 per page)\n", - "Processing 806 of 1000 on page 1 - (1000 per page)\n", - "Processing 807 of 1000 on page 1 - (1000 per page)\n", - "Processing 808 of 1000 on page 1 - (1000 per page)\n", - "Processing 809 of 1000 on page 1 - (1000 per page)\n", - "Processing 810 of 1000 on page 1 - (1000 per page)\n", - "Processing 811 of 1000 on page 1 - (1000 per page)\n", - "Processing 812 of 1000 on page 1 - (1000 per page)\n", - "Processing 813 of 1000 on page 1 - (1000 per page)\n", - "Processing 814 of 1000 on page 1 - (1000 per page)\n", - "Processing 815 of 1000 on page 1 - (1000 per page)\n", - "Processing 816 of 1000 on page 1 - (1000 per page)\n", - "Processing 817 of 1000 on page 1 - (1000 per page)\n", - "Processing 818 of 1000 on page 1 - (1000 per page)\n", - "Processing 819 of 1000 on page 1 - (1000 per page)\n", - "Processing 820 of 1000 on page 1 - (1000 per page)\n", - "Processing 821 of 1000 on page 1 - (1000 per page)\n", - "Processing 822 of 1000 on page 1 - (1000 per page)\n", - "Processing 823 of 1000 on page 1 - (1000 per page)\n", - "Processing 824 of 1000 on page 1 - (1000 per page)\n", - "Processing 825 of 1000 on page 1 - (1000 per page)\n", - "Processing 826 of 1000 on page 1 - (1000 per page)\n", - "Processing 827 of 1000 on page 1 - (1000 per page)\n", - "Processing 828 of 1000 on page 1 - (1000 per page)\n", - "Processing 829 of 1000 on page 1 - (1000 per page)\n", - "Processing 830 of 1000 on page 1 - (1000 per page)\n", - "Processing 831 of 1000 on page 1 - (1000 per page)\n", - "Processing 832 of 1000 on page 1 - (1000 per page)\n", - "Processing 833 of 1000 on page 1 - (1000 per page)\n", - "Processing 834 of 1000 on page 1 - (1000 per page)\n", - "Processing 835 of 1000 on page 1 - (1000 per page)\n", - "Processing 836 of 1000 on page 1 - (1000 per page)\n", - "Processing 837 of 1000 on page 1 - (1000 per page)\n", - "Processing 838 of 1000 on page 1 - (1000 per page)\n", - "Processing 839 of 1000 on page 1 - (1000 per page)\n", - "Processing 840 of 1000 on page 1 - (1000 per page)\n", - "Processing 841 of 1000 on page 1 - (1000 per page)\n", - "Processing 842 of 1000 on page 1 - (1000 per page)\n", - "Processing 843 of 1000 on page 1 - (1000 per page)\n", - "Processing 844 of 1000 on page 1 - (1000 per page)\n", - "Processing 845 of 1000 on page 1 - (1000 per page)\n", - "Processing 846 of 1000 on page 1 - (1000 per page)\n", - "Processing 847 of 1000 on page 1 - (1000 per page)\n", - "Processing 848 of 1000 on page 1 - (1000 per page)\n", - "Processing 849 of 1000 on page 1 - (1000 per page)\n", - "Processing 850 of 1000 on page 1 - (1000 per page)\n", - "Processing 851 of 1000 on page 1 - (1000 per page)\n", - "Processing 852 of 1000 on page 1 - (1000 per page)\n", - "Processing 853 of 1000 on page 1 - (1000 per page)\n", - "Processing 854 of 1000 on page 1 - (1000 per page)\n", - "Processing 855 of 1000 on page 1 - (1000 per page)\n", - "Processing 856 of 1000 on page 1 - (1000 per page)\n", - "Processing 857 of 1000 on page 1 - (1000 per page)\n", - "Processing 858 of 1000 on page 1 - (1000 per page)\n", - "Processing 859 of 1000 on page 1 - (1000 per page)\n", - "Processing 860 of 1000 on page 1 - (1000 per page)\n", - "Processing 861 of 1000 on page 1 - (1000 per page)\n", - "Processing 862 of 1000 on page 1 - (1000 per page)\n", - "Processing 863 of 1000 on page 1 - (1000 per page)\n", - "Processing 864 of 1000 on page 1 - (1000 per page)\n", - "Processing 865 of 1000 on page 1 - (1000 per page)\n", - "Processing 866 of 1000 on page 1 - (1000 per page)\n", - "Processing 867 of 1000 on page 1 - (1000 per page)\n", - "Processing 868 of 1000 on page 1 - (1000 per page)\n", - "Processing 869 of 1000 on page 1 - (1000 per page)\n", - "Processing 870 of 1000 on page 1 - (1000 per page)\n", - "Processing 871 of 1000 on page 1 - (1000 per page)\n", - "Processing 872 of 1000 on page 1 - (1000 per page)\n", - "Processing 873 of 1000 on page 1 - (1000 per page)\n", - "Processing 874 of 1000 on page 1 - (1000 per page)\n", - "Processing 875 of 1000 on page 1 - (1000 per page)\n", - "Processing 876 of 1000 on page 1 - (1000 per page)\n", - "Processing 877 of 1000 on page 1 - (1000 per page)\n", - "Processing 878 of 1000 on page 1 - (1000 per page)\n", - "Processing 879 of 1000 on page 1 - (1000 per page)\n", - "Processing 880 of 1000 on page 1 - (1000 per page)\n", - "Processing 881 of 1000 on page 1 - (1000 per page)\n", - "Processing 882 of 1000 on page 1 - (1000 per page)\n", - "Processing 883 of 1000 on page 1 - (1000 per page)\n", - "Processing 884 of 1000 on page 1 - (1000 per page)\n", - "Processing 885 of 1000 on page 1 - (1000 per page)\n", - "Processing 886 of 1000 on page 1 - (1000 per page)\n", - "Processing 887 of 1000 on page 1 - (1000 per page)\n", - "Processing 888 of 1000 on page 1 - (1000 per page)\n", - "Processing 889 of 1000 on page 1 - (1000 per page)\n", - "Processing 890 of 1000 on page 1 - (1000 per page)\n", - "Processing 891 of 1000 on page 1 - (1000 per page)\n", - "Processing 892 of 1000 on page 1 - (1000 per page)\n", - "Processing 893 of 1000 on page 1 - (1000 per page)\n", - "Processing 894 of 1000 on page 1 - (1000 per page)\n", - "Processing 895 of 1000 on page 1 - (1000 per page)\n", - "Processing 896 of 1000 on page 1 - (1000 per page)\n", - "Processing 897 of 1000 on page 1 - (1000 per page)\n", - "Processing 898 of 1000 on page 1 - (1000 per page)\n", - "Processing 899 of 1000 on page 1 - (1000 per page)\n", - "Processing 900 of 1000 on page 1 - (1000 per page)\n", - "Processing 901 of 1000 on page 1 - (1000 per page)\n", - "Processing 902 of 1000 on page 1 - (1000 per page)\n", - "Processing 903 of 1000 on page 1 - (1000 per page)\n", - "Processing 904 of 1000 on page 1 - (1000 per page)\n", - "Processing 905 of 1000 on page 1 - (1000 per page)\n", - "Processing 906 of 1000 on page 1 - (1000 per page)\n", - "Processing 907 of 1000 on page 1 - (1000 per page)\n", - "Processing 908 of 1000 on page 1 - (1000 per page)\n", - "Processing 909 of 1000 on page 1 - (1000 per page)\n", - "Processing 910 of 1000 on page 1 - (1000 per page)\n", - "Processing 911 of 1000 on page 1 - (1000 per page)\n", - "Processing 912 of 1000 on page 1 - (1000 per page)\n", - "Processing 913 of 1000 on page 1 - (1000 per page)\n", - "Processing 914 of 1000 on page 1 - (1000 per page)\n", - "Processing 915 of 1000 on page 1 - (1000 per page)\n", - "Processing 916 of 1000 on page 1 - (1000 per page)\n", - "Processing 917 of 1000 on page 1 - (1000 per page)\n", - "Processing 918 of 1000 on page 1 - (1000 per page)\n", - "Processing 919 of 1000 on page 1 - (1000 per page)\n", - "Processing 920 of 1000 on page 1 - (1000 per page)\n", - "Processing 921 of 1000 on page 1 - (1000 per page)\n", - "Processing 922 of 1000 on page 1 - (1000 per page)\n", - "Processing 923 of 1000 on page 1 - (1000 per page)\n", - "Processing 924 of 1000 on page 1 - (1000 per page)\n", - "Processing 925 of 1000 on page 1 - (1000 per page)\n", - "Processing 926 of 1000 on page 1 - (1000 per page)\n", - "Processing 927 of 1000 on page 1 - (1000 per page)\n", - "Processing 928 of 1000 on page 1 - (1000 per page)\n", - "Processing 929 of 1000 on page 1 - (1000 per page)\n", - "Processing 930 of 1000 on page 1 - (1000 per page)\n", - "Processing 931 of 1000 on page 1 - (1000 per page)\n", - "Processing 932 of 1000 on page 1 - (1000 per page)\n", - "Processing 933 of 1000 on page 1 - (1000 per page)\n", - "Processing 934 of 1000 on page 1 - (1000 per page)\n", - "Processing 935 of 1000 on page 1 - (1000 per page)\n", - "Processing 936 of 1000 on page 1 - (1000 per page)\n", - "Processing 937 of 1000 on page 1 - (1000 per page)\n", - "Processing 938 of 1000 on page 1 - (1000 per page)\n", - "Processing 939 of 1000 on page 1 - (1000 per page)\n", - "Processing 940 of 1000 on page 1 - (1000 per page)\n", - "Processing 941 of 1000 on page 1 - (1000 per page)\n", - "Processing 942 of 1000 on page 1 - (1000 per page)\n", - "Processing 943 of 1000 on page 1 - (1000 per page)\n", - "Processing 944 of 1000 on page 1 - (1000 per page)\n", - "Processing 945 of 1000 on page 1 - (1000 per page)\n", - "Processing 946 of 1000 on page 1 - (1000 per page)\n", - "Processing 947 of 1000 on page 1 - (1000 per page)\n", - "Processing 948 of 1000 on page 1 - (1000 per page)\n", - "Processing 949 of 1000 on page 1 - (1000 per page)\n", - "Processing 950 of 1000 on page 1 - (1000 per page)\n", - "Processing 951 of 1000 on page 1 - (1000 per page)\n", - "Processing 952 of 1000 on page 1 - (1000 per page)\n", - "Processing 953 of 1000 on page 1 - (1000 per page)\n", - "Processing 954 of 1000 on page 1 - (1000 per page)\n", - "Processing 955 of 1000 on page 1 - (1000 per page)\n", - "Processing 956 of 1000 on page 1 - (1000 per page)\n", - "Processing 957 of 1000 on page 1 - (1000 per page)\n", - "Processing 958 of 1000 on page 1 - (1000 per page)\n", - "Processing 959 of 1000 on page 1 - (1000 per page)\n", - "Processing 960 of 1000 on page 1 - (1000 per page)\n", - "Processing 961 of 1000 on page 1 - (1000 per page)\n", - "Processing 962 of 1000 on page 1 - (1000 per page)\n", - "Processing 963 of 1000 on page 1 - (1000 per page)\n", - "Processing 964 of 1000 on page 1 - (1000 per page)\n", - "Processing 965 of 1000 on page 1 - (1000 per page)\n", - "Processing 966 of 1000 on page 1 - (1000 per page)\n", - "Processing 967 of 1000 on page 1 - (1000 per page)\n", - "Processing 968 of 1000 on page 1 - (1000 per page)\n", - "Processing 969 of 1000 on page 1 - (1000 per page)\n", - "Processing 970 of 1000 on page 1 - (1000 per page)\n", - "Processing 971 of 1000 on page 1 - (1000 per page)\n", - "Processing 972 of 1000 on page 1 - (1000 per page)\n", - "Processing 973 of 1000 on page 1 - (1000 per page)\n", - "Processing 974 of 1000 on page 1 - (1000 per page)\n", - "Processing 975 of 1000 on page 1 - (1000 per page)\n", - "Processing 976 of 1000 on page 1 - (1000 per page)\n", - "Processing 977 of 1000 on page 1 - (1000 per page)\n", - "Processing 978 of 1000 on page 1 - (1000 per page)\n", - "Processing 979 of 1000 on page 1 - (1000 per page)\n", - "Processing 980 of 1000 on page 1 - (1000 per page)\n", - "Processing 981 of 1000 on page 1 - (1000 per page)\n", - "Processing 982 of 1000 on page 1 - (1000 per page)\n", - "Processing 983 of 1000 on page 1 - (1000 per page)\n", - "Processing 984 of 1000 on page 1 - (1000 per page)\n", - "Processing 985 of 1000 on page 1 - (1000 per page)\n", - "Processing 986 of 1000 on page 1 - (1000 per page)\n", - "Processing 987 of 1000 on page 1 - (1000 per page)\n", - "Processing 988 of 1000 on page 1 - (1000 per page)\n", - "Processing 989 of 1000 on page 1 - (1000 per page)\n", - "Processing 990 of 1000 on page 1 - (1000 per page)\n", - "Processing 991 of 1000 on page 1 - (1000 per page)\n", - "Processing 992 of 1000 on page 1 - (1000 per page)\n", - "Processing 993 of 1000 on page 1 - (1000 per page)\n", - "Processing 994 of 1000 on page 1 - (1000 per page)\n", - "Processing 995 of 1000 on page 1 - (1000 per page)\n", - "Processing 996 of 1000 on page 1 - (1000 per page)\n", - "Processing 997 of 1000 on page 1 - (1000 per page)\n", - "Processing 998 of 1000 on page 1 - (1000 per page)\n", - "Processing 999 of 1000 on page 1 - (1000 per page)\n", - "Processing 1000 of 1000 on page 1 - (1000 per page)\n", - "Processing 1 of 1000 on page 2 - (1000 per page)\n", - "Processing 2 of 1000 on page 2 - (1000 per page)\n", - "Processing 3 of 1000 on page 2 - (1000 per page)\n", - "Processing 4 of 1000 on page 2 - (1000 per page)\n", - "Processing 5 of 1000 on page 2 - (1000 per page)\n", - "Processing 6 of 1000 on page 2 - (1000 per page)\n", - "Processing 7 of 1000 on page 2 - (1000 per page)\n", - "Processing 8 of 1000 on page 2 - (1000 per page)\n", - "Processing 9 of 1000 on page 2 - (1000 per page)\n", - "Processing 10 of 1000 on page 2 - (1000 per page)\n", - "Processing 11 of 1000 on page 2 - (1000 per page)\n", - "Processing 12 of 1000 on page 2 - (1000 per page)\n", - "Processing 13 of 1000 on page 2 - (1000 per page)\n", - "Processing 14 of 1000 on page 2 - (1000 per page)\n", - "Processing 15 of 1000 on page 2 - (1000 per page)\n", - "Processing 16 of 1000 on page 2 - (1000 per page)\n", - "Processing 17 of 1000 on page 2 - (1000 per page)\n", - "Processing 18 of 1000 on page 2 - (1000 per page)\n", - "Processing 19 of 1000 on page 2 - (1000 per page)\n", - "Processing 20 of 1000 on page 2 - (1000 per page)\n", - "Processing 21 of 1000 on page 2 - (1000 per page)\n", - "Processing 22 of 1000 on page 2 - (1000 per page)\n", - "Processing 23 of 1000 on page 2 - (1000 per page)\n", - "Processing 24 of 1000 on page 2 - (1000 per page)\n", - "Processing 25 of 1000 on page 2 - (1000 per page)\n", - "Processing 26 of 1000 on page 2 - (1000 per page)\n", - "Processing 27 of 1000 on page 2 - (1000 per page)\n", - "Processing 28 of 1000 on page 2 - (1000 per page)\n", - "Processing 29 of 1000 on page 2 - (1000 per page)\n", - "Processing 30 of 1000 on page 2 - (1000 per page)\n", - "Processing 31 of 1000 on page 2 - (1000 per page)\n", - "Processing 32 of 1000 on page 2 - (1000 per page)\n", - "Processing 33 of 1000 on page 2 - (1000 per page)\n", - "Processing 34 of 1000 on page 2 - (1000 per page)\n", - "Processing 35 of 1000 on page 2 - (1000 per page)\n", - "Processing 36 of 1000 on page 2 - (1000 per page)\n", - "Processing 37 of 1000 on page 2 - (1000 per page)\n", - "Processing 38 of 1000 on page 2 - (1000 per page)\n", - "Processing 39 of 1000 on page 2 - (1000 per page)\n", - "Processing 40 of 1000 on page 2 - (1000 per page)\n", - "Processing 41 of 1000 on page 2 - (1000 per page)\n", - "Processing 42 of 1000 on page 2 - (1000 per page)\n", - "Processing 43 of 1000 on page 2 - (1000 per page)\n", - "Processing 44 of 1000 on page 2 - (1000 per page)\n", - "Processing 45 of 1000 on page 2 - (1000 per page)\n", - "Processing 46 of 1000 on page 2 - (1000 per page)\n", - "Processing 47 of 1000 on page 2 - (1000 per page)\n", - "Processing 48 of 1000 on page 2 - (1000 per page)\n", - "Processing 49 of 1000 on page 2 - (1000 per page)\n", - "Processing 50 of 1000 on page 2 - (1000 per page)\n", - "Processing 51 of 1000 on page 2 - (1000 per page)\n", - "Processing 52 of 1000 on page 2 - (1000 per page)\n", - "Processing 53 of 1000 on page 2 - (1000 per page)\n", - "Processing 54 of 1000 on page 2 - (1000 per page)\n", - "Processing 55 of 1000 on page 2 - (1000 per page)\n", - "Processing 56 of 1000 on page 2 - (1000 per page)\n", - "Processing 57 of 1000 on page 2 - (1000 per page)\n", - "Processing 58 of 1000 on page 2 - (1000 per page)\n", - "Processing 59 of 1000 on page 2 - (1000 per page)\n", - "Processing 60 of 1000 on page 2 - (1000 per page)\n", - "Processing 61 of 1000 on page 2 - (1000 per page)\n", - "Processing 62 of 1000 on page 2 - (1000 per page)\n", - "Processing 63 of 1000 on page 2 - (1000 per page)\n", - "Processing 64 of 1000 on page 2 - (1000 per page)\n", - "Processing 65 of 1000 on page 2 - (1000 per page)\n", - "Processing 66 of 1000 on page 2 - (1000 per page)\n", - "Processing 67 of 1000 on page 2 - (1000 per page)\n", - "Processing 68 of 1000 on page 2 - (1000 per page)\n", - "Processing 69 of 1000 on page 2 - (1000 per page)\n", - "Processing 70 of 1000 on page 2 - (1000 per page)\n", - "Processing 71 of 1000 on page 2 - (1000 per page)\n", - "Processing 72 of 1000 on page 2 - (1000 per page)\n", - "Processing 73 of 1000 on page 2 - (1000 per page)\n", - "Processing 74 of 1000 on page 2 - (1000 per page)\n", - "Processing 75 of 1000 on page 2 - (1000 per page)\n", - "Processing 76 of 1000 on page 2 - (1000 per page)\n", - "Processing 77 of 1000 on page 2 - (1000 per page)\n", - "Processing 78 of 1000 on page 2 - (1000 per page)\n", - "Processing 79 of 1000 on page 2 - (1000 per page)\n", - "Processing 80 of 1000 on page 2 - (1000 per page)\n", - "Processing 81 of 1000 on page 2 - (1000 per page)\n", - "Processing 82 of 1000 on page 2 - (1000 per page)\n", - "Processing 83 of 1000 on page 2 - (1000 per page)\n", - "Processing 84 of 1000 on page 2 - (1000 per page)\n", - "Processing 85 of 1000 on page 2 - (1000 per page)\n", - "Processing 86 of 1000 on page 2 - (1000 per page)\n", - "Processing 87 of 1000 on page 2 - (1000 per page)\n", - "Processing 88 of 1000 on page 2 - (1000 per page)\n", - "Processing 89 of 1000 on page 2 - (1000 per page)\n", - "Processing 90 of 1000 on page 2 - (1000 per page)\n", - "Processing 91 of 1000 on page 2 - (1000 per page)\n", - "Processing 92 of 1000 on page 2 - (1000 per page)\n", - "Processing 93 of 1000 on page 2 - (1000 per page)\n", - "Processing 94 of 1000 on page 2 - (1000 per page)\n", - "Processing 95 of 1000 on page 2 - (1000 per page)\n", - "Processing 96 of 1000 on page 2 - (1000 per page)\n", - "Processing 97 of 1000 on page 2 - (1000 per page)\n", - "Processing 98 of 1000 on page 2 - (1000 per page)\n", - "Processing 99 of 1000 on page 2 - (1000 per page)\n", - "Processing 100 of 1000 on page 2 - (1000 per page)\n", - "Processing 101 of 1000 on page 2 - (1000 per page)\n", - "Processing 102 of 1000 on page 2 - (1000 per page)\n", - "Processing 103 of 1000 on page 2 - (1000 per page)\n", - "Processing 104 of 1000 on page 2 - (1000 per page)\n", - "Processing 105 of 1000 on page 2 - (1000 per page)\n", - "Processing 106 of 1000 on page 2 - (1000 per page)\n", - "Processing 107 of 1000 on page 2 - (1000 per page)\n", - "Processing 108 of 1000 on page 2 - (1000 per page)\n", - "Processing 109 of 1000 on page 2 - (1000 per page)\n", - "Processing 110 of 1000 on page 2 - (1000 per page)\n", - "Processing 111 of 1000 on page 2 - (1000 per page)\n", - "Processing 112 of 1000 on page 2 - (1000 per page)\n", - "Processing 113 of 1000 on page 2 - (1000 per page)\n", - "Processing 114 of 1000 on page 2 - (1000 per page)\n", - "Processing 115 of 1000 on page 2 - (1000 per page)\n", - "Processing 116 of 1000 on page 2 - (1000 per page)\n", - "Processing 117 of 1000 on page 2 - (1000 per page)\n", - "Processing 118 of 1000 on page 2 - (1000 per page)\n", - "Processing 119 of 1000 on page 2 - (1000 per page)\n", - "Processing 120 of 1000 on page 2 - (1000 per page)\n", - "Processing 121 of 1000 on page 2 - (1000 per page)\n", - "Processing 122 of 1000 on page 2 - (1000 per page)\n", - "Processing 123 of 1000 on page 2 - (1000 per page)\n", - "Processing 124 of 1000 on page 2 - (1000 per page)\n", - "Processing 125 of 1000 on page 2 - (1000 per page)\n", - "Processing 126 of 1000 on page 2 - (1000 per page)\n", - "Processing 127 of 1000 on page 2 - (1000 per page)\n", - "Processing 128 of 1000 on page 2 - (1000 per page)\n", - "Processing 129 of 1000 on page 2 - (1000 per page)\n", - "Processing 130 of 1000 on page 2 - (1000 per page)\n", - "Processing 131 of 1000 on page 2 - (1000 per page)\n", - "Processing 132 of 1000 on page 2 - (1000 per page)\n", - "Processing 133 of 1000 on page 2 - (1000 per page)\n", - "Processing 134 of 1000 on page 2 - (1000 per page)\n", - "Processing 135 of 1000 on page 2 - (1000 per page)\n", - "Processing 136 of 1000 on page 2 - (1000 per page)\n", - "Processing 137 of 1000 on page 2 - (1000 per page)\n", - "Processing 138 of 1000 on page 2 - (1000 per page)\n", - "Processing 139 of 1000 on page 2 - (1000 per page)\n", - "Processing 140 of 1000 on page 2 - (1000 per page)\n", - "Processing 141 of 1000 on page 2 - (1000 per page)\n", - "Processing 142 of 1000 on page 2 - (1000 per page)\n", - "Processing 143 of 1000 on page 2 - (1000 per page)\n", - "Processing 144 of 1000 on page 2 - (1000 per page)\n", - "Processing 145 of 1000 on page 2 - (1000 per page)\n", - "Processing 146 of 1000 on page 2 - (1000 per page)\n", - "Processing 147 of 1000 on page 2 - (1000 per page)\n", - "Processing 148 of 1000 on page 2 - (1000 per page)\n", - "Processing 149 of 1000 on page 2 - (1000 per page)\n", - "Processing 150 of 1000 on page 2 - (1000 per page)\n", - "Processing 151 of 1000 on page 2 - (1000 per page)\n", - "Processing 152 of 1000 on page 2 - (1000 per page)\n", - "Processing 153 of 1000 on page 2 - (1000 per page)\n", - "Processing 154 of 1000 on page 2 - (1000 per page)\n", - "Processing 155 of 1000 on page 2 - (1000 per page)\n", - "Processing 156 of 1000 on page 2 - (1000 per page)\n", - "Processing 157 of 1000 on page 2 - (1000 per page)\n", - "Processing 158 of 1000 on page 2 - (1000 per page)\n", - "Processing 159 of 1000 on page 2 - (1000 per page)\n", - "Processing 160 of 1000 on page 2 - (1000 per page)\n", - "Processing 161 of 1000 on page 2 - (1000 per page)\n", - "Processing 162 of 1000 on page 2 - (1000 per page)\n", - "Processing 163 of 1000 on page 2 - (1000 per page)\n", - "Processing 164 of 1000 on page 2 - (1000 per page)\n", - "Processing 165 of 1000 on page 2 - (1000 per page)\n", - "Processing 166 of 1000 on page 2 - (1000 per page)\n", - "Processing 167 of 1000 on page 2 - (1000 per page)\n", - "Processing 168 of 1000 on page 2 - (1000 per page)\n", - "Processing 169 of 1000 on page 2 - (1000 per page)\n", - "Processing 170 of 1000 on page 2 - (1000 per page)\n", - "Processing 171 of 1000 on page 2 - (1000 per page)\n", - "Processing 172 of 1000 on page 2 - (1000 per page)\n", - "Processing 173 of 1000 on page 2 - (1000 per page)\n", - "Processing 174 of 1000 on page 2 - (1000 per page)\n", - "Processing 175 of 1000 on page 2 - (1000 per page)\n", - "Processing 176 of 1000 on page 2 - (1000 per page)\n", - "Processing 177 of 1000 on page 2 - (1000 per page)\n", - "Processing 178 of 1000 on page 2 - (1000 per page)\n", - "Processing 179 of 1000 on page 2 - (1000 per page)\n", - "Processing 180 of 1000 on page 2 - (1000 per page)\n", - "Processing 181 of 1000 on page 2 - (1000 per page)\n", - "Processing 182 of 1000 on page 2 - (1000 per page)\n", - "Processing 183 of 1000 on page 2 - (1000 per page)\n", - "Processing 184 of 1000 on page 2 - (1000 per page)\n", - "Processing 185 of 1000 on page 2 - (1000 per page)\n", - "Processing 186 of 1000 on page 2 - (1000 per page)\n", - "Processing 187 of 1000 on page 2 - (1000 per page)\n", - "Processing 188 of 1000 on page 2 - (1000 per page)\n", - "Processing 189 of 1000 on page 2 - (1000 per page)\n", - "Processing 190 of 1000 on page 2 - (1000 per page)\n", - "Processing 191 of 1000 on page 2 - (1000 per page)\n", - "Processing 192 of 1000 on page 2 - (1000 per page)\n", - "Processing 193 of 1000 on page 2 - (1000 per page)\n", - "Processing 194 of 1000 on page 2 - (1000 per page)\n", - "Processing 195 of 1000 on page 2 - (1000 per page)\n", - "Processing 196 of 1000 on page 2 - (1000 per page)\n", - "Processing 197 of 1000 on page 2 - (1000 per page)\n", - "Processing 198 of 1000 on page 2 - (1000 per page)\n", - "Processing 199 of 1000 on page 2 - (1000 per page)\n", - "Processing 200 of 1000 on page 2 - (1000 per page)\n", - "Processing 201 of 1000 on page 2 - (1000 per page)\n", - "Processing 202 of 1000 on page 2 - (1000 per page)\n", - "Processing 203 of 1000 on page 2 - (1000 per page)\n", - "Processing 204 of 1000 on page 2 - (1000 per page)\n", - "Processing 205 of 1000 on page 2 - (1000 per page)\n", - "Processing 206 of 1000 on page 2 - (1000 per page)\n", - "Processing 207 of 1000 on page 2 - (1000 per page)\n", - "Processing 208 of 1000 on page 2 - (1000 per page)\n", - "Processing 209 of 1000 on page 2 - (1000 per page)\n", - "Processing 210 of 1000 on page 2 - (1000 per page)\n", - "Processing 211 of 1000 on page 2 - (1000 per page)\n", - "Processing 212 of 1000 on page 2 - (1000 per page)\n", - "Processing 213 of 1000 on page 2 - (1000 per page)\n", - "Processing 214 of 1000 on page 2 - (1000 per page)\n", - "Processing 215 of 1000 on page 2 - (1000 per page)\n", - "Processing 216 of 1000 on page 2 - (1000 per page)\n", - "Processing 217 of 1000 on page 2 - (1000 per page)\n", - "Processing 218 of 1000 on page 2 - (1000 per page)\n", - "Processing 219 of 1000 on page 2 - (1000 per page)\n", - "Processing 220 of 1000 on page 2 - (1000 per page)\n", - "Processing 221 of 1000 on page 2 - (1000 per page)\n", - "Processing 222 of 1000 on page 2 - (1000 per page)\n", - "Processing 223 of 1000 on page 2 - (1000 per page)\n", - "Processing 224 of 1000 on page 2 - (1000 per page)\n", - "Processing 225 of 1000 on page 2 - (1000 per page)\n", - "Processing 226 of 1000 on page 2 - (1000 per page)\n", - "Processing 227 of 1000 on page 2 - (1000 per page)\n", - "Processing 228 of 1000 on page 2 - (1000 per page)\n", - "Processing 229 of 1000 on page 2 - (1000 per page)\n", - "Processing 230 of 1000 on page 2 - (1000 per page)\n", - "Processing 231 of 1000 on page 2 - (1000 per page)\n", - "Processing 232 of 1000 on page 2 - (1000 per page)\n", - "Processing 233 of 1000 on page 2 - (1000 per page)\n", - "Processing 234 of 1000 on page 2 - (1000 per page)\n", - "Processing 235 of 1000 on page 2 - (1000 per page)\n", - "Processing 236 of 1000 on page 2 - (1000 per page)\n", - "Processing 237 of 1000 on page 2 - (1000 per page)\n", - "Processing 238 of 1000 on page 2 - (1000 per page)\n", - "Processing 239 of 1000 on page 2 - (1000 per page)\n", - "Processing 240 of 1000 on page 2 - (1000 per page)\n", - "Processing 241 of 1000 on page 2 - (1000 per page)\n", - "Processing 242 of 1000 on page 2 - (1000 per page)\n", - "Processing 243 of 1000 on page 2 - (1000 per page)\n", - "Processing 244 of 1000 on page 2 - (1000 per page)\n", - "Processing 245 of 1000 on page 2 - (1000 per page)\n", - "Processing 246 of 1000 on page 2 - (1000 per page)\n", - "Processing 247 of 1000 on page 2 - (1000 per page)\n", - "Processing 248 of 1000 on page 2 - (1000 per page)\n", - "Processing 249 of 1000 on page 2 - (1000 per page)\n", - "Processing 250 of 1000 on page 2 - (1000 per page)\n", - "Processing 251 of 1000 on page 2 - (1000 per page)\n", - "Processing 252 of 1000 on page 2 - (1000 per page)\n", - "Processing 253 of 1000 on page 2 - (1000 per page)\n", - "Processing 254 of 1000 on page 2 - (1000 per page)\n", - "Processing 255 of 1000 on page 2 - (1000 per page)\n", - "Processing 256 of 1000 on page 2 - (1000 per page)\n", - "Processing 257 of 1000 on page 2 - (1000 per page)\n", - "Processing 258 of 1000 on page 2 - (1000 per page)\n", - "Processing 259 of 1000 on page 2 - (1000 per page)\n", - "Processing 260 of 1000 on page 2 - (1000 per page)\n", - "Processing 261 of 1000 on page 2 - (1000 per page)\n", - "Processing 262 of 1000 on page 2 - (1000 per page)\n", - "Processing 263 of 1000 on page 2 - (1000 per page)\n", - "Processing 264 of 1000 on page 2 - (1000 per page)\n", - "Processing 265 of 1000 on page 2 - (1000 per page)\n", - "Processing 266 of 1000 on page 2 - (1000 per page)\n", - "Processing 267 of 1000 on page 2 - (1000 per page)\n", - "Processing 268 of 1000 on page 2 - (1000 per page)\n", - "Processing 269 of 1000 on page 2 - (1000 per page)\n", - "Processing 270 of 1000 on page 2 - (1000 per page)\n", - "Processing 271 of 1000 on page 2 - (1000 per page)\n", - "Processing 272 of 1000 on page 2 - (1000 per page)\n", - "Processing 273 of 1000 on page 2 - (1000 per page)\n", - "Processing 274 of 1000 on page 2 - (1000 per page)\n", - "Processing 275 of 1000 on page 2 - (1000 per page)\n", - "Processing 276 of 1000 on page 2 - (1000 per page)\n", - "Processing 277 of 1000 on page 2 - (1000 per page)\n", - "Processing 278 of 1000 on page 2 - (1000 per page)\n", - "Processing 279 of 1000 on page 2 - (1000 per page)\n", - "Processing 280 of 1000 on page 2 - (1000 per page)\n", - "Processing 281 of 1000 on page 2 - (1000 per page)\n", - "Processing 282 of 1000 on page 2 - (1000 per page)\n", - "Processing 283 of 1000 on page 2 - (1000 per page)\n", - "Processing 284 of 1000 on page 2 - (1000 per page)\n", - "Processing 285 of 1000 on page 2 - (1000 per page)\n", - "Processing 286 of 1000 on page 2 - (1000 per page)\n", - "Processing 287 of 1000 on page 2 - (1000 per page)\n", - "Processing 288 of 1000 on page 2 - (1000 per page)\n", - "Processing 289 of 1000 on page 2 - (1000 per page)\n", - "Processing 290 of 1000 on page 2 - (1000 per page)\n", - "Processing 291 of 1000 on page 2 - (1000 per page)\n", - "Processing 292 of 1000 on page 2 - (1000 per page)\n", - "Processing 293 of 1000 on page 2 - (1000 per page)\n", - "Processing 294 of 1000 on page 2 - (1000 per page)\n", - "Processing 295 of 1000 on page 2 - (1000 per page)\n", - "Processing 296 of 1000 on page 2 - (1000 per page)\n", - "Processing 297 of 1000 on page 2 - (1000 per page)\n", - "Processing 298 of 1000 on page 2 - (1000 per page)\n", - "Processing 299 of 1000 on page 2 - (1000 per page)\n", - "Processing 300 of 1000 on page 2 - (1000 per page)\n", - "Processing 301 of 1000 on page 2 - (1000 per page)\n", - "Processing 302 of 1000 on page 2 - (1000 per page)\n", - "Processing 303 of 1000 on page 2 - (1000 per page)\n", - "Processing 304 of 1000 on page 2 - (1000 per page)\n", - "Processing 305 of 1000 on page 2 - (1000 per page)\n", - "Processing 306 of 1000 on page 2 - (1000 per page)\n", - "Processing 307 of 1000 on page 2 - (1000 per page)\n", - "Processing 308 of 1000 on page 2 - (1000 per page)\n", - "Processing 309 of 1000 on page 2 - (1000 per page)\n", - "Processing 310 of 1000 on page 2 - (1000 per page)\n", - "Processing 311 of 1000 on page 2 - (1000 per page)\n", - "Processing 312 of 1000 on page 2 - (1000 per page)\n", - "Processing 313 of 1000 on page 2 - (1000 per page)\n", - "Processing 314 of 1000 on page 2 - (1000 per page)\n", - "Processing 315 of 1000 on page 2 - (1000 per page)\n", - "Processing 316 of 1000 on page 2 - (1000 per page)\n", - "Processing 317 of 1000 on page 2 - (1000 per page)\n", - "Processing 318 of 1000 on page 2 - (1000 per page)\n", - "Processing 319 of 1000 on page 2 - (1000 per page)\n", - "Processing 320 of 1000 on page 2 - (1000 per page)\n", - "Processing 321 of 1000 on page 2 - (1000 per page)\n", - "Processing 322 of 1000 on page 2 - (1000 per page)\n", - "Processing 323 of 1000 on page 2 - (1000 per page)\n", - "Processing 324 of 1000 on page 2 - (1000 per page)\n", - "Processing 325 of 1000 on page 2 - (1000 per page)\n", - "Processing 326 of 1000 on page 2 - (1000 per page)\n", - "Processing 327 of 1000 on page 2 - (1000 per page)\n", - "Processing 328 of 1000 on page 2 - (1000 per page)\n", - "Processing 329 of 1000 on page 2 - (1000 per page)\n", - "Processing 330 of 1000 on page 2 - (1000 per page)\n", - "Processing 331 of 1000 on page 2 - (1000 per page)\n", - "Processing 332 of 1000 on page 2 - (1000 per page)\n", - "Processing 333 of 1000 on page 2 - (1000 per page)\n", - "Processing 334 of 1000 on page 2 - (1000 per page)\n", - "Processing 335 of 1000 on page 2 - (1000 per page)\n", - "Processing 336 of 1000 on page 2 - (1000 per page)\n", - "Processing 337 of 1000 on page 2 - (1000 per page)\n", - "Processing 338 of 1000 on page 2 - (1000 per page)\n", - "Processing 339 of 1000 on page 2 - (1000 per page)\n", - "Processing 340 of 1000 on page 2 - (1000 per page)\n", - "Processing 341 of 1000 on page 2 - (1000 per page)\n", - "Processing 342 of 1000 on page 2 - (1000 per page)\n", - "Processing 343 of 1000 on page 2 - (1000 per page)\n", - "Processing 344 of 1000 on page 2 - (1000 per page)\n", - "Processing 345 of 1000 on page 2 - (1000 per page)\n", - "Processing 346 of 1000 on page 2 - (1000 per page)\n", - "Processing 347 of 1000 on page 2 - (1000 per page)\n", - "Processing 348 of 1000 on page 2 - (1000 per page)\n", - "Processing 349 of 1000 on page 2 - (1000 per page)\n", - "Processing 350 of 1000 on page 2 - (1000 per page)\n", - "Processing 351 of 1000 on page 2 - (1000 per page)\n", - "Processing 352 of 1000 on page 2 - (1000 per page)\n", - "Processing 353 of 1000 on page 2 - (1000 per page)\n", - "Processing 354 of 1000 on page 2 - (1000 per page)\n", - "Processing 355 of 1000 on page 2 - (1000 per page)\n", - "Processing 356 of 1000 on page 2 - (1000 per page)\n", - "Processing 357 of 1000 on page 2 - (1000 per page)\n", - "Processing 358 of 1000 on page 2 - (1000 per page)\n", - "Processing 359 of 1000 on page 2 - (1000 per page)\n", - "Processing 360 of 1000 on page 2 - (1000 per page)\n", - "Processing 361 of 1000 on page 2 - (1000 per page)\n", - "Processing 362 of 1000 on page 2 - (1000 per page)\n", - "Processing 363 of 1000 on page 2 - (1000 per page)\n", - "Processing 364 of 1000 on page 2 - (1000 per page)\n", - "Processing 365 of 1000 on page 2 - (1000 per page)\n", - "Processing 366 of 1000 on page 2 - (1000 per page)\n", - "Processing 367 of 1000 on page 2 - (1000 per page)\n", - "Processing 368 of 1000 on page 2 - (1000 per page)\n", - "Processing 369 of 1000 on page 2 - (1000 per page)\n", - "Processing 370 of 1000 on page 2 - (1000 per page)\n", - "Processing 371 of 1000 on page 2 - (1000 per page)\n", - "Processing 372 of 1000 on page 2 - (1000 per page)\n", - "Processing 373 of 1000 on page 2 - (1000 per page)\n", - "Processing 374 of 1000 on page 2 - (1000 per page)\n", - "Processing 375 of 1000 on page 2 - (1000 per page)\n", - "Processing 376 of 1000 on page 2 - (1000 per page)\n", - "Processing 377 of 1000 on page 2 - (1000 per page)\n", - "Processing 378 of 1000 on page 2 - (1000 per page)\n", - "Processing 379 of 1000 on page 2 - (1000 per page)\n", - "Processing 380 of 1000 on page 2 - (1000 per page)\n", - "Processing 381 of 1000 on page 2 - (1000 per page)\n", - "Processing 382 of 1000 on page 2 - (1000 per page)\n", - "Processing 383 of 1000 on page 2 - (1000 per page)\n", - "Processing 384 of 1000 on page 2 - (1000 per page)\n", - "Processing 385 of 1000 on page 2 - (1000 per page)\n", - "Processing 386 of 1000 on page 2 - (1000 per page)\n", - "Processing 387 of 1000 on page 2 - (1000 per page)\n", - "Processing 388 of 1000 on page 2 - (1000 per page)\n", - "Processing 389 of 1000 on page 2 - (1000 per page)\n", - "Processing 390 of 1000 on page 2 - (1000 per page)\n", - "Processing 391 of 1000 on page 2 - (1000 per page)\n", - "Processing 392 of 1000 on page 2 - (1000 per page)\n", - "Processing 393 of 1000 on page 2 - (1000 per page)\n", - "Processing 394 of 1000 on page 2 - (1000 per page)\n", - "Processing 395 of 1000 on page 2 - (1000 per page)\n", - "Processing 396 of 1000 on page 2 - (1000 per page)\n", - "Processing 397 of 1000 on page 2 - (1000 per page)\n", - "Processing 398 of 1000 on page 2 - (1000 per page)\n", - "Processing 399 of 1000 on page 2 - (1000 per page)\n", - "Processing 400 of 1000 on page 2 - (1000 per page)\n", - "Processing 401 of 1000 on page 2 - (1000 per page)\n", - "Processing 402 of 1000 on page 2 - (1000 per page)\n", - "Processing 403 of 1000 on page 2 - (1000 per page)\n", - "Processing 404 of 1000 on page 2 - (1000 per page)\n", - "Processing 405 of 1000 on page 2 - (1000 per page)\n", - "Processing 406 of 1000 on page 2 - (1000 per page)\n", - "Processing 407 of 1000 on page 2 - (1000 per page)\n", - "Processing 408 of 1000 on page 2 - (1000 per page)\n", - "Processing 409 of 1000 on page 2 - (1000 per page)\n", - "Processing 410 of 1000 on page 2 - (1000 per page)\n", - "Processing 411 of 1000 on page 2 - (1000 per page)\n", - "Processing 412 of 1000 on page 2 - (1000 per page)\n", - "Processing 413 of 1000 on page 2 - (1000 per page)\n", - "Processing 414 of 1000 on page 2 - (1000 per page)\n", - "Processing 415 of 1000 on page 2 - (1000 per page)\n", - "Processing 416 of 1000 on page 2 - (1000 per page)\n", - "Processing 417 of 1000 on page 2 - (1000 per page)\n", - "Processing 418 of 1000 on page 2 - (1000 per page)\n", - "Processing 419 of 1000 on page 2 - (1000 per page)\n", - "Processing 420 of 1000 on page 2 - (1000 per page)\n", - "Processing 421 of 1000 on page 2 - (1000 per page)\n", - "Processing 422 of 1000 on page 2 - (1000 per page)\n", - "Processing 423 of 1000 on page 2 - (1000 per page)\n", - "Processing 424 of 1000 on page 2 - (1000 per page)\n", - "Processing 425 of 1000 on page 2 - (1000 per page)\n", - "Processing 426 of 1000 on page 2 - (1000 per page)\n", - "Processing 427 of 1000 on page 2 - (1000 per page)\n", - "Processing 428 of 1000 on page 2 - (1000 per page)\n", - "Processing 429 of 1000 on page 2 - (1000 per page)\n", - "Processing 430 of 1000 on page 2 - (1000 per page)\n", - "Processing 431 of 1000 on page 2 - (1000 per page)\n", - "Processing 432 of 1000 on page 2 - (1000 per page)\n", - "Processing 433 of 1000 on page 2 - (1000 per page)\n", - "Processing 434 of 1000 on page 2 - (1000 per page)\n", - "Processing 435 of 1000 on page 2 - (1000 per page)\n", - "Processing 436 of 1000 on page 2 - (1000 per page)\n", - "Processing 437 of 1000 on page 2 - (1000 per page)\n", - "Processing 438 of 1000 on page 2 - (1000 per page)\n", - "Processing 439 of 1000 on page 2 - (1000 per page)\n", - "Processing 440 of 1000 on page 2 - (1000 per page)\n", - "Processing 441 of 1000 on page 2 - (1000 per page)\n", - "Processing 442 of 1000 on page 2 - (1000 per page)\n", - "Processing 443 of 1000 on page 2 - (1000 per page)\n", - "Processing 444 of 1000 on page 2 - (1000 per page)\n", - "Processing 445 of 1000 on page 2 - (1000 per page)\n", - "Processing 446 of 1000 on page 2 - (1000 per page)\n", - "Processing 447 of 1000 on page 2 - (1000 per page)\n", - "Processing 448 of 1000 on page 2 - (1000 per page)\n", - "Processing 449 of 1000 on page 2 - (1000 per page)\n", - "Processing 450 of 1000 on page 2 - (1000 per page)\n", - "Processing 451 of 1000 on page 2 - (1000 per page)\n", - "Processing 452 of 1000 on page 2 - (1000 per page)\n", - "Processing 453 of 1000 on page 2 - (1000 per page)\n", - "Processing 454 of 1000 on page 2 - (1000 per page)\n", - "Processing 455 of 1000 on page 2 - (1000 per page)\n", - "Processing 456 of 1000 on page 2 - (1000 per page)\n", - "Processing 457 of 1000 on page 2 - (1000 per page)\n", - "Processing 458 of 1000 on page 2 - (1000 per page)\n", - "Processing 459 of 1000 on page 2 - (1000 per page)\n", - "Processing 460 of 1000 on page 2 - (1000 per page)\n", - "Processing 461 of 1000 on page 2 - (1000 per page)\n", - "Processing 462 of 1000 on page 2 - (1000 per page)\n", - "Processing 463 of 1000 on page 2 - (1000 per page)\n", - "Processing 464 of 1000 on page 2 - (1000 per page)\n", - "Processing 465 of 1000 on page 2 - (1000 per page)\n", - "Processing 466 of 1000 on page 2 - (1000 per page)\n", - "Processing 467 of 1000 on page 2 - (1000 per page)\n", - "Processing 468 of 1000 on page 2 - (1000 per page)\n", - "Processing 469 of 1000 on page 2 - (1000 per page)\n", - "Processing 470 of 1000 on page 2 - (1000 per page)\n", - "Processing 471 of 1000 on page 2 - (1000 per page)\n", - "Processing 472 of 1000 on page 2 - (1000 per page)\n", - "Processing 473 of 1000 on page 2 - (1000 per page)\n", - "Processing 474 of 1000 on page 2 - (1000 per page)\n", - "Processing 475 of 1000 on page 2 - (1000 per page)\n", - "Processing 476 of 1000 on page 2 - (1000 per page)\n", - "Processing 477 of 1000 on page 2 - (1000 per page)\n", - "Processing 478 of 1000 on page 2 - (1000 per page)\n", - "Processing 479 of 1000 on page 2 - (1000 per page)\n", - "Processing 480 of 1000 on page 2 - (1000 per page)\n", - "Processing 481 of 1000 on page 2 - (1000 per page)\n", - "Processing 482 of 1000 on page 2 - (1000 per page)\n", - "Processing 483 of 1000 on page 2 - (1000 per page)\n", - "Processing 484 of 1000 on page 2 - (1000 per page)\n", - "Processing 485 of 1000 on page 2 - (1000 per page)\n", - "Processing 486 of 1000 on page 2 - (1000 per page)\n", - "Processing 487 of 1000 on page 2 - (1000 per page)\n", - "Processing 488 of 1000 on page 2 - (1000 per page)\n", - "Processing 489 of 1000 on page 2 - (1000 per page)\n", - "Processing 490 of 1000 on page 2 - (1000 per page)\n", - "Processing 491 of 1000 on page 2 - (1000 per page)\n", - "Processing 492 of 1000 on page 2 - (1000 per page)\n", - "Processing 493 of 1000 on page 2 - (1000 per page)\n", - "Processing 494 of 1000 on page 2 - (1000 per page)\n", - "Processing 495 of 1000 on page 2 - (1000 per page)\n", - "Processing 496 of 1000 on page 2 - (1000 per page)\n", - "Processing 497 of 1000 on page 2 - (1000 per page)\n", - "Processing 498 of 1000 on page 2 - (1000 per page)\n", - "Processing 499 of 1000 on page 2 - (1000 per page)\n", - "Processing 500 of 1000 on page 2 - (1000 per page)\n", - "Processing 501 of 1000 on page 2 - (1000 per page)\n", - "Processing 502 of 1000 on page 2 - (1000 per page)\n", - "Processing 503 of 1000 on page 2 - (1000 per page)\n", - "Processing 504 of 1000 on page 2 - (1000 per page)\n", - "Processing 505 of 1000 on page 2 - (1000 per page)\n", - "Processing 506 of 1000 on page 2 - (1000 per page)\n", - "Processing 507 of 1000 on page 2 - (1000 per page)\n", - "Processing 508 of 1000 on page 2 - (1000 per page)\n", - "Processing 509 of 1000 on page 2 - (1000 per page)\n", - "Processing 510 of 1000 on page 2 - (1000 per page)\n", - "Processing 511 of 1000 on page 2 - (1000 per page)\n", - "Processing 512 of 1000 on page 2 - (1000 per page)\n", - "Processing 513 of 1000 on page 2 - (1000 per page)\n", - "Processing 514 of 1000 on page 2 - (1000 per page)\n", - "Processing 515 of 1000 on page 2 - (1000 per page)\n", - "Processing 516 of 1000 on page 2 - (1000 per page)\n", - "Processing 517 of 1000 on page 2 - (1000 per page)\n", - "Processing 518 of 1000 on page 2 - (1000 per page)\n", - "Processing 519 of 1000 on page 2 - (1000 per page)\n", - "Processing 520 of 1000 on page 2 - (1000 per page)\n", - "Processing 521 of 1000 on page 2 - (1000 per page)\n", - "Processing 522 of 1000 on page 2 - (1000 per page)\n", - "Processing 523 of 1000 on page 2 - (1000 per page)\n", - "Processing 524 of 1000 on page 2 - (1000 per page)\n", - "Processing 525 of 1000 on page 2 - (1000 per page)\n", - "Processing 526 of 1000 on page 2 - (1000 per page)\n", - "Processing 527 of 1000 on page 2 - (1000 per page)\n", - "Processing 528 of 1000 on page 2 - (1000 per page)\n", - "Processing 529 of 1000 on page 2 - (1000 per page)\n", - "Processing 530 of 1000 on page 2 - (1000 per page)\n", - "Processing 531 of 1000 on page 2 - (1000 per page)\n", - "Processing 532 of 1000 on page 2 - (1000 per page)\n", - "Processing 533 of 1000 on page 2 - (1000 per page)\n", - "Processing 534 of 1000 on page 2 - (1000 per page)\n", - "Processing 535 of 1000 on page 2 - (1000 per page)\n", - "Processing 536 of 1000 on page 2 - (1000 per page)\n", - "Processing 537 of 1000 on page 2 - (1000 per page)\n", - "Processing 538 of 1000 on page 2 - (1000 per page)\n", - "Processing 539 of 1000 on page 2 - (1000 per page)\n", - "Processing 540 of 1000 on page 2 - (1000 per page)\n", - "Processing 541 of 1000 on page 2 - (1000 per page)\n", - "Processing 542 of 1000 on page 2 - (1000 per page)\n", - "Processing 543 of 1000 on page 2 - (1000 per page)\n", - "Processing 544 of 1000 on page 2 - (1000 per page)\n", - "Processing 545 of 1000 on page 2 - (1000 per page)\n", - "Processing 546 of 1000 on page 2 - (1000 per page)\n", - "Processing 547 of 1000 on page 2 - (1000 per page)\n", - "Processing 548 of 1000 on page 2 - (1000 per page)\n", - "Processing 549 of 1000 on page 2 - (1000 per page)\n", - "Processing 550 of 1000 on page 2 - (1000 per page)\n", - "Processing 551 of 1000 on page 2 - (1000 per page)\n", - "Processing 552 of 1000 on page 2 - (1000 per page)\n", - "Processing 553 of 1000 on page 2 - (1000 per page)\n", - "Processing 554 of 1000 on page 2 - (1000 per page)\n", - "Processing 555 of 1000 on page 2 - (1000 per page)\n", - "Processing 556 of 1000 on page 2 - (1000 per page)\n", - "Processing 557 of 1000 on page 2 - (1000 per page)\n", - "Processing 558 of 1000 on page 2 - (1000 per page)\n", - "Processing 559 of 1000 on page 2 - (1000 per page)\n", - "Processing 560 of 1000 on page 2 - (1000 per page)\n", - "Processing 561 of 1000 on page 2 - (1000 per page)\n", - "Processing 562 of 1000 on page 2 - (1000 per page)\n", - "Processing 563 of 1000 on page 2 - (1000 per page)\n", - "Processing 564 of 1000 on page 2 - (1000 per page)\n", - "Processing 565 of 1000 on page 2 - (1000 per page)\n", - "Processing 566 of 1000 on page 2 - (1000 per page)\n", - "Processing 567 of 1000 on page 2 - (1000 per page)\n", - "Processing 568 of 1000 on page 2 - (1000 per page)\n", - "Processing 569 of 1000 on page 2 - (1000 per page)\n", - "Processing 570 of 1000 on page 2 - (1000 per page)\n", - "Processing 571 of 1000 on page 2 - (1000 per page)\n", - "Processing 572 of 1000 on page 2 - (1000 per page)\n", - "Processing 573 of 1000 on page 2 - (1000 per page)\n", - "Processing 574 of 1000 on page 2 - (1000 per page)\n", - "Processing 575 of 1000 on page 2 - (1000 per page)\n", - "Processing 576 of 1000 on page 2 - (1000 per page)\n", - "Processing 577 of 1000 on page 2 - (1000 per page)\n", - "Processing 578 of 1000 on page 2 - (1000 per page)\n", - "Processing 579 of 1000 on page 2 - (1000 per page)\n", - "Processing 580 of 1000 on page 2 - (1000 per page)\n", - "Processing 581 of 1000 on page 2 - (1000 per page)\n", - "Processing 582 of 1000 on page 2 - (1000 per page)\n", - "Processing 583 of 1000 on page 2 - (1000 per page)\n", - "Processing 584 of 1000 on page 2 - (1000 per page)\n", - "Processing 585 of 1000 on page 2 - (1000 per page)\n", - "Processing 586 of 1000 on page 2 - (1000 per page)\n", - "Processing 587 of 1000 on page 2 - (1000 per page)\n", - "Processing 588 of 1000 on page 2 - (1000 per page)\n", - "Processing 589 of 1000 on page 2 - (1000 per page)\n", - "Processing 590 of 1000 on page 2 - (1000 per page)\n", - "Processing 591 of 1000 on page 2 - (1000 per page)\n", - "Processing 592 of 1000 on page 2 - (1000 per page)\n", - "Processing 593 of 1000 on page 2 - (1000 per page)\n", - "Processing 594 of 1000 on page 2 - (1000 per page)\n", - "Processing 595 of 1000 on page 2 - (1000 per page)\n", - "Processing 596 of 1000 on page 2 - (1000 per page)\n", - "Processing 597 of 1000 on page 2 - (1000 per page)\n", - "Processing 598 of 1000 on page 2 - (1000 per page)\n", - "Processing 599 of 1000 on page 2 - (1000 per page)\n", - "Processing 600 of 1000 on page 2 - (1000 per page)\n", - "Processing 601 of 1000 on page 2 - (1000 per page)\n", - "Processing 602 of 1000 on page 2 - (1000 per page)\n", - "Processing 603 of 1000 on page 2 - (1000 per page)\n", - "Processing 604 of 1000 on page 2 - (1000 per page)\n", - "Processing 605 of 1000 on page 2 - (1000 per page)\n", - "Processing 606 of 1000 on page 2 - (1000 per page)\n", - "Processing 607 of 1000 on page 2 - (1000 per page)\n", - "Processing 608 of 1000 on page 2 - (1000 per page)\n", - "Processing 609 of 1000 on page 2 - (1000 per page)\n", - "Processing 610 of 1000 on page 2 - (1000 per page)\n", - "Processing 611 of 1000 on page 2 - (1000 per page)\n", - "Processing 612 of 1000 on page 2 - (1000 per page)\n", - "Processing 613 of 1000 on page 2 - (1000 per page)\n", - "Processing 614 of 1000 on page 2 - (1000 per page)\n", - "Processing 615 of 1000 on page 2 - (1000 per page)\n", - "Processing 616 of 1000 on page 2 - (1000 per page)\n", - "Processing 617 of 1000 on page 2 - (1000 per page)\n", - "Processing 618 of 1000 on page 2 - (1000 per page)\n", - "Processing 619 of 1000 on page 2 - (1000 per page)\n", - "Processing 620 of 1000 on page 2 - (1000 per page)\n", - "Processing 621 of 1000 on page 2 - (1000 per page)\n", - "Processing 622 of 1000 on page 2 - (1000 per page)\n", - "Processing 623 of 1000 on page 2 - (1000 per page)\n", - "Processing 624 of 1000 on page 2 - (1000 per page)\n", - "Processing 625 of 1000 on page 2 - (1000 per page)\n", - "Processing 626 of 1000 on page 2 - (1000 per page)\n", - "Processing 627 of 1000 on page 2 - (1000 per page)\n", - "Processing 628 of 1000 on page 2 - (1000 per page)\n", - "Processing 629 of 1000 on page 2 - (1000 per page)\n", - "Processing 630 of 1000 on page 2 - (1000 per page)\n", - "Processing 631 of 1000 on page 2 - (1000 per page)\n", - "Processing 632 of 1000 on page 2 - (1000 per page)\n", - "Processing 633 of 1000 on page 2 - (1000 per page)\n", - "Processing 634 of 1000 on page 2 - (1000 per page)\n", - "Processing 635 of 1000 on page 2 - (1000 per page)\n", - "Processing 636 of 1000 on page 2 - (1000 per page)\n", - "Processing 637 of 1000 on page 2 - (1000 per page)\n", - "Processing 638 of 1000 on page 2 - (1000 per page)\n", - "Processing 639 of 1000 on page 2 - (1000 per page)\n", - "Processing 640 of 1000 on page 2 - (1000 per page)\n", - "Processing 641 of 1000 on page 2 - (1000 per page)\n", - "Processing 642 of 1000 on page 2 - (1000 per page)\n", - "Processing 643 of 1000 on page 2 - (1000 per page)\n", - "Processing 644 of 1000 on page 2 - (1000 per page)\n", - "Processing 645 of 1000 on page 2 - (1000 per page)\n", - "Processing 646 of 1000 on page 2 - (1000 per page)\n", - "Processing 647 of 1000 on page 2 - (1000 per page)\n", - "Processing 648 of 1000 on page 2 - (1000 per page)\n", - "Processing 649 of 1000 on page 2 - (1000 per page)\n", - "Processing 650 of 1000 on page 2 - (1000 per page)\n", - "Processing 651 of 1000 on page 2 - (1000 per page)\n", - "Processing 652 of 1000 on page 2 - (1000 per page)\n", - "Processing 653 of 1000 on page 2 - (1000 per page)\n", - "Processing 654 of 1000 on page 2 - (1000 per page)\n", - "Processing 655 of 1000 on page 2 - (1000 per page)\n", - "Processing 656 of 1000 on page 2 - (1000 per page)\n", - "Processing 657 of 1000 on page 2 - (1000 per page)\n", - "Processing 658 of 1000 on page 2 - (1000 per page)\n", - "Processing 659 of 1000 on page 2 - (1000 per page)\n", - "Processing 660 of 1000 on page 2 - (1000 per page)\n", - "Processing 661 of 1000 on page 2 - (1000 per page)\n", - "Processing 662 of 1000 on page 2 - (1000 per page)\n", - "Processing 663 of 1000 on page 2 - (1000 per page)\n", - "Processing 664 of 1000 on page 2 - (1000 per page)\n", - "Processing 665 of 1000 on page 2 - (1000 per page)\n", - "Processing 666 of 1000 on page 2 - (1000 per page)\n", - "Processing 667 of 1000 on page 2 - (1000 per page)\n", - "Processing 668 of 1000 on page 2 - (1000 per page)\n", - "Processing 669 of 1000 on page 2 - (1000 per page)\n", - "Processing 670 of 1000 on page 2 - (1000 per page)\n", - "Processing 671 of 1000 on page 2 - (1000 per page)\n", - "Processing 672 of 1000 on page 2 - (1000 per page)\n", - "Processing 673 of 1000 on page 2 - (1000 per page)\n", - "Processing 674 of 1000 on page 2 - (1000 per page)\n", - "Processing 675 of 1000 on page 2 - (1000 per page)\n", - "Processing 676 of 1000 on page 2 - (1000 per page)\n", - "Processing 677 of 1000 on page 2 - (1000 per page)\n", - "Processing 678 of 1000 on page 2 - (1000 per page)\n", - "Processing 679 of 1000 on page 2 - (1000 per page)\n", - "Processing 680 of 1000 on page 2 - (1000 per page)\n", - "Processing 681 of 1000 on page 2 - (1000 per page)\n", - "Processing 682 of 1000 on page 2 - (1000 per page)\n", - "Processing 683 of 1000 on page 2 - (1000 per page)\n", - "Processing 684 of 1000 on page 2 - (1000 per page)\n", - "Processing 685 of 1000 on page 2 - (1000 per page)\n", - "Processing 686 of 1000 on page 2 - (1000 per page)\n", - "Processing 687 of 1000 on page 2 - (1000 per page)\n", - "Processing 688 of 1000 on page 2 - (1000 per page)\n", - "Processing 689 of 1000 on page 2 - (1000 per page)\n", - "Processing 690 of 1000 on page 2 - (1000 per page)\n", - "Processing 691 of 1000 on page 2 - (1000 per page)\n", - "Processing 692 of 1000 on page 2 - (1000 per page)\n", - "Processing 693 of 1000 on page 2 - (1000 per page)\n", - "Processing 694 of 1000 on page 2 - (1000 per page)\n", - "Processing 695 of 1000 on page 2 - (1000 per page)\n", - "Processing 696 of 1000 on page 2 - (1000 per page)\n", - "Processing 697 of 1000 on page 2 - (1000 per page)\n", - "Processing 698 of 1000 on page 2 - (1000 per page)\n", - "Processing 699 of 1000 on page 2 - (1000 per page)\n", - "Processing 700 of 1000 on page 2 - (1000 per page)\n", - "Processing 701 of 1000 on page 2 - (1000 per page)\n", - "Processing 702 of 1000 on page 2 - (1000 per page)\n", - "Processing 703 of 1000 on page 2 - (1000 per page)\n", - "Processing 704 of 1000 on page 2 - (1000 per page)\n", - "Processing 705 of 1000 on page 2 - (1000 per page)\n", - "Processing 706 of 1000 on page 2 - (1000 per page)\n", - "Processing 707 of 1000 on page 2 - (1000 per page)\n", - "Processing 708 of 1000 on page 2 - (1000 per page)\n", - "Processing 709 of 1000 on page 2 - (1000 per page)\n", - "Processing 710 of 1000 on page 2 - (1000 per page)\n", - "Processing 711 of 1000 on page 2 - (1000 per page)\n", - "Processing 712 of 1000 on page 2 - (1000 per page)\n", - "Processing 713 of 1000 on page 2 - (1000 per page)\n", - "Processing 714 of 1000 on page 2 - (1000 per page)\n", - "Processing 715 of 1000 on page 2 - (1000 per page)\n", - "Processing 716 of 1000 on page 2 - (1000 per page)\n", - "Processing 717 of 1000 on page 2 - (1000 per page)\n", - "Processing 718 of 1000 on page 2 - (1000 per page)\n", - "Processing 719 of 1000 on page 2 - (1000 per page)\n", - "Processing 720 of 1000 on page 2 - (1000 per page)\n", - "Processing 721 of 1000 on page 2 - (1000 per page)\n", - "Processing 722 of 1000 on page 2 - (1000 per page)\n", - "Processing 723 of 1000 on page 2 - (1000 per page)\n", - "Processing 724 of 1000 on page 2 - (1000 per page)\n", - "Processing 725 of 1000 on page 2 - (1000 per page)\n", - "Processing 726 of 1000 on page 2 - (1000 per page)\n", - "Processing 727 of 1000 on page 2 - (1000 per page)\n", - "Processing 728 of 1000 on page 2 - (1000 per page)\n", - "Processing 729 of 1000 on page 2 - (1000 per page)\n", - "Processing 730 of 1000 on page 2 - (1000 per page)\n", - "Processing 731 of 1000 on page 2 - (1000 per page)\n", - "Processing 732 of 1000 on page 2 - (1000 per page)\n", - "Processing 733 of 1000 on page 2 - (1000 per page)\n", - "Processing 734 of 1000 on page 2 - (1000 per page)\n", - "Processing 735 of 1000 on page 2 - (1000 per page)\n", - "Processing 736 of 1000 on page 2 - (1000 per page)\n", - "Processing 737 of 1000 on page 2 - (1000 per page)\n", - "Processing 738 of 1000 on page 2 - (1000 per page)\n", - "Processing 739 of 1000 on page 2 - (1000 per page)\n", - "Processing 740 of 1000 on page 2 - (1000 per page)\n", - "Processing 741 of 1000 on page 2 - (1000 per page)\n", - "Processing 742 of 1000 on page 2 - (1000 per page)\n", - "Processing 743 of 1000 on page 2 - (1000 per page)\n", - "Processing 744 of 1000 on page 2 - (1000 per page)\n", - "Processing 745 of 1000 on page 2 - (1000 per page)\n", - "Processing 746 of 1000 on page 2 - (1000 per page)\n", - "Processing 747 of 1000 on page 2 - (1000 per page)\n", - "Processing 748 of 1000 on page 2 - (1000 per page)\n", - "Processing 749 of 1000 on page 2 - (1000 per page)\n", - "Processing 750 of 1000 on page 2 - (1000 per page)\n", - "Processing 751 of 1000 on page 2 - (1000 per page)\n", - "Processing 752 of 1000 on page 2 - (1000 per page)\n", - "Processing 753 of 1000 on page 2 - (1000 per page)\n", - "Processing 754 of 1000 on page 2 - (1000 per page)\n", - "Processing 755 of 1000 on page 2 - (1000 per page)\n", - "Processing 756 of 1000 on page 2 - (1000 per page)\n", - "Processing 757 of 1000 on page 2 - (1000 per page)\n", - "Processing 758 of 1000 on page 2 - (1000 per page)\n", - "Processing 759 of 1000 on page 2 - (1000 per page)\n", - "Processing 760 of 1000 on page 2 - (1000 per page)\n", - "Processing 761 of 1000 on page 2 - (1000 per page)\n", - "Processing 762 of 1000 on page 2 - (1000 per page)\n", - "Processing 763 of 1000 on page 2 - (1000 per page)\n", - "Processing 764 of 1000 on page 2 - (1000 per page)\n", - "Processing 765 of 1000 on page 2 - (1000 per page)\n", - "Processing 766 of 1000 on page 2 - (1000 per page)\n", - "Processing 767 of 1000 on page 2 - (1000 per page)\n", - "Processing 768 of 1000 on page 2 - (1000 per page)\n", - "Processing 769 of 1000 on page 2 - (1000 per page)\n", - "Processing 770 of 1000 on page 2 - (1000 per page)\n", - "Processing 771 of 1000 on page 2 - (1000 per page)\n", - "Processing 772 of 1000 on page 2 - (1000 per page)\n", - "Processing 773 of 1000 on page 2 - (1000 per page)\n", - "Processing 774 of 1000 on page 2 - (1000 per page)\n", - "Processing 775 of 1000 on page 2 - (1000 per page)\n", - "Processing 776 of 1000 on page 2 - (1000 per page)\n", - "Processing 777 of 1000 on page 2 - (1000 per page)\n", - "Processing 778 of 1000 on page 2 - (1000 per page)\n", - "Processing 779 of 1000 on page 2 - (1000 per page)\n", - "Processing 780 of 1000 on page 2 - (1000 per page)\n", - "Processing 781 of 1000 on page 2 - (1000 per page)\n", - "Processing 782 of 1000 on page 2 - (1000 per page)\n", - "Processing 783 of 1000 on page 2 - (1000 per page)\n", - "Processing 784 of 1000 on page 2 - (1000 per page)\n", - "Processing 785 of 1000 on page 2 - (1000 per page)\n", - "Processing 786 of 1000 on page 2 - (1000 per page)\n", - "Processing 787 of 1000 on page 2 - (1000 per page)\n", - "Processing 788 of 1000 on page 2 - (1000 per page)\n", - "Processing 789 of 1000 on page 2 - (1000 per page)\n", - "Processing 790 of 1000 on page 2 - (1000 per page)\n", - "Processing 791 of 1000 on page 2 - (1000 per page)\n", - "Processing 792 of 1000 on page 2 - (1000 per page)\n", - "Processing 793 of 1000 on page 2 - (1000 per page)\n", - "Processing 794 of 1000 on page 2 - (1000 per page)\n", - "Processing 795 of 1000 on page 2 - (1000 per page)\n", - "Processing 796 of 1000 on page 2 - (1000 per page)\n", - "Processing 797 of 1000 on page 2 - (1000 per page)\n", - "Processing 798 of 1000 on page 2 - (1000 per page)\n", - "Processing 799 of 1000 on page 2 - (1000 per page)\n", - "Processing 800 of 1000 on page 2 - (1000 per page)\n", - "Processing 801 of 1000 on page 2 - (1000 per page)\n", - "Processing 802 of 1000 on page 2 - (1000 per page)\n", - "Processing 803 of 1000 on page 2 - (1000 per page)\n", - "Processing 804 of 1000 on page 2 - (1000 per page)\n", - "Processing 805 of 1000 on page 2 - (1000 per page)\n", - "Processing 806 of 1000 on page 2 - (1000 per page)\n", - "Processing 807 of 1000 on page 2 - (1000 per page)\n", - "Processing 808 of 1000 on page 2 - (1000 per page)\n", - "Processing 809 of 1000 on page 2 - (1000 per page)\n", - "Processing 810 of 1000 on page 2 - (1000 per page)\n", - "Processing 811 of 1000 on page 2 - (1000 per page)\n", - "Processing 812 of 1000 on page 2 - (1000 per page)\n", - "Processing 813 of 1000 on page 2 - (1000 per page)\n", - "Processing 814 of 1000 on page 2 - (1000 per page)\n", - "Processing 815 of 1000 on page 2 - (1000 per page)\n", - "Processing 816 of 1000 on page 2 - (1000 per page)\n", - "Processing 817 of 1000 on page 2 - (1000 per page)\n", - "Processing 818 of 1000 on page 2 - (1000 per page)\n", - "Processing 819 of 1000 on page 2 - (1000 per page)\n", - "Processing 820 of 1000 on page 2 - (1000 per page)\n", - "Processing 821 of 1000 on page 2 - (1000 per page)\n", - "Processing 822 of 1000 on page 2 - (1000 per page)\n", - "Processing 823 of 1000 on page 2 - (1000 per page)\n", - "Processing 824 of 1000 on page 2 - (1000 per page)\n", - "Processing 825 of 1000 on page 2 - (1000 per page)\n", - "Processing 826 of 1000 on page 2 - (1000 per page)\n", - "Processing 827 of 1000 on page 2 - (1000 per page)\n", - "Processing 828 of 1000 on page 2 - (1000 per page)\n", - "Processing 829 of 1000 on page 2 - (1000 per page)\n", - "Processing 830 of 1000 on page 2 - (1000 per page)\n", - "Processing 831 of 1000 on page 2 - (1000 per page)\n", - "Processing 832 of 1000 on page 2 - (1000 per page)\n", - "Processing 833 of 1000 on page 2 - (1000 per page)\n", - "Processing 834 of 1000 on page 2 - (1000 per page)\n", - "Processing 835 of 1000 on page 2 - (1000 per page)\n", - "Processing 836 of 1000 on page 2 - (1000 per page)\n", - "Processing 837 of 1000 on page 2 - (1000 per page)\n", - "Processing 838 of 1000 on page 2 - (1000 per page)\n", - "Processing 839 of 1000 on page 2 - (1000 per page)\n", - "Processing 840 of 1000 on page 2 - (1000 per page)\n", - "Processing 841 of 1000 on page 2 - (1000 per page)\n", - "Processing 842 of 1000 on page 2 - (1000 per page)\n", - "Processing 843 of 1000 on page 2 - (1000 per page)\n", - "Processing 844 of 1000 on page 2 - (1000 per page)\n", - "Processing 845 of 1000 on page 2 - (1000 per page)\n", - "Processing 846 of 1000 on page 2 - (1000 per page)\n", - "Processing 847 of 1000 on page 2 - (1000 per page)\n", - "Processing 848 of 1000 on page 2 - (1000 per page)\n", - "Processing 849 of 1000 on page 2 - (1000 per page)\n", - "Processing 850 of 1000 on page 2 - (1000 per page)\n", - "Processing 851 of 1000 on page 2 - (1000 per page)\n", - "Processing 852 of 1000 on page 2 - (1000 per page)\n", - "Processing 853 of 1000 on page 2 - (1000 per page)\n", - "Processing 854 of 1000 on page 2 - (1000 per page)\n", - "Processing 855 of 1000 on page 2 - (1000 per page)\n", - "Processing 856 of 1000 on page 2 - (1000 per page)\n", - "Processing 857 of 1000 on page 2 - (1000 per page)\n", - "Processing 858 of 1000 on page 2 - (1000 per page)\n", - "Processing 859 of 1000 on page 2 - (1000 per page)\n", - "Processing 860 of 1000 on page 2 - (1000 per page)\n", - "Processing 861 of 1000 on page 2 - (1000 per page)\n", - "Processing 862 of 1000 on page 2 - (1000 per page)\n", - "Processing 863 of 1000 on page 2 - (1000 per page)\n", - "Processing 864 of 1000 on page 2 - (1000 per page)\n", - "Processing 865 of 1000 on page 2 - (1000 per page)\n", - "Processing 866 of 1000 on page 2 - (1000 per page)\n", - "Processing 867 of 1000 on page 2 - (1000 per page)\n", - "Processing 868 of 1000 on page 2 - (1000 per page)\n", - "Processing 869 of 1000 on page 2 - (1000 per page)\n", - "Processing 870 of 1000 on page 2 - (1000 per page)\n", - "Processing 871 of 1000 on page 2 - (1000 per page)\n", - "Processing 872 of 1000 on page 2 - (1000 per page)\n", - "Processing 873 of 1000 on page 2 - (1000 per page)\n", - "Processing 874 of 1000 on page 2 - (1000 per page)\n", - "Processing 875 of 1000 on page 2 - (1000 per page)\n", - "Processing 876 of 1000 on page 2 - (1000 per page)\n", - "Processing 877 of 1000 on page 2 - (1000 per page)\n", - "Processing 878 of 1000 on page 2 - (1000 per page)\n", - "Processing 879 of 1000 on page 2 - (1000 per page)\n", - "Processing 880 of 1000 on page 2 - (1000 per page)\n", - "Processing 881 of 1000 on page 2 - (1000 per page)\n", - "Processing 882 of 1000 on page 2 - (1000 per page)\n", - "Processing 883 of 1000 on page 2 - (1000 per page)\n", - "Processing 884 of 1000 on page 2 - (1000 per page)\n", - "Processing 885 of 1000 on page 2 - (1000 per page)\n", - "Processing 886 of 1000 on page 2 - (1000 per page)\n", - "Processing 887 of 1000 on page 2 - (1000 per page)\n", - "Processing 888 of 1000 on page 2 - (1000 per page)\n", - "Processing 889 of 1000 on page 2 - (1000 per page)\n", - "Processing 890 of 1000 on page 2 - (1000 per page)\n", - "Processing 891 of 1000 on page 2 - (1000 per page)\n", - "Processing 892 of 1000 on page 2 - (1000 per page)\n", - "Processing 893 of 1000 on page 2 - (1000 per page)\n", - "Processing 894 of 1000 on page 2 - (1000 per page)\n", - "Processing 895 of 1000 on page 2 - (1000 per page)\n", - "Processing 896 of 1000 on page 2 - (1000 per page)\n", - "Processing 897 of 1000 on page 2 - (1000 per page)\n", - "Processing 898 of 1000 on page 2 - (1000 per page)\n", - "Processing 899 of 1000 on page 2 - (1000 per page)\n", - "Processing 900 of 1000 on page 2 - (1000 per page)\n", - "Processing 901 of 1000 on page 2 - (1000 per page)\n", - "Processing 902 of 1000 on page 2 - (1000 per page)\n", - "Processing 903 of 1000 on page 2 - (1000 per page)\n", - "Processing 904 of 1000 on page 2 - (1000 per page)\n", - "Processing 905 of 1000 on page 2 - (1000 per page)\n", - "Processing 906 of 1000 on page 2 - (1000 per page)\n", - "Processing 907 of 1000 on page 2 - (1000 per page)\n", - "Processing 908 of 1000 on page 2 - (1000 per page)\n", - "Processing 909 of 1000 on page 2 - (1000 per page)\n", - "Processing 910 of 1000 on page 2 - (1000 per page)\n", - "Processing 911 of 1000 on page 2 - (1000 per page)\n", - "Processing 912 of 1000 on page 2 - (1000 per page)\n", - "Processing 913 of 1000 on page 2 - (1000 per page)\n", - "Processing 914 of 1000 on page 2 - (1000 per page)\n", - "Processing 915 of 1000 on page 2 - (1000 per page)\n", - "Processing 916 of 1000 on page 2 - (1000 per page)\n", - "Processing 917 of 1000 on page 2 - (1000 per page)\n", - "Processing 918 of 1000 on page 2 - (1000 per page)\n", - "Processing 919 of 1000 on page 2 - (1000 per page)\n", - "Processing 920 of 1000 on page 2 - (1000 per page)\n", - "Processing 921 of 1000 on page 2 - (1000 per page)\n", - "Processing 922 of 1000 on page 2 - (1000 per page)\n", - "Processing 923 of 1000 on page 2 - (1000 per page)\n", - "Processing 924 of 1000 on page 2 - (1000 per page)\n", - "Processing 925 of 1000 on page 2 - (1000 per page)\n", - "Processing 926 of 1000 on page 2 - (1000 per page)\n", - "Processing 927 of 1000 on page 2 - (1000 per page)\n", - "Processing 928 of 1000 on page 2 - (1000 per page)\n", - "Processing 929 of 1000 on page 2 - (1000 per page)\n", - "Processing 930 of 1000 on page 2 - (1000 per page)\n", - "Processing 931 of 1000 on page 2 - (1000 per page)\n", - "Processing 932 of 1000 on page 2 - (1000 per page)\n", - "Processing 933 of 1000 on page 2 - (1000 per page)\n", - "Processing 934 of 1000 on page 2 - (1000 per page)\n", - "Processing 935 of 1000 on page 2 - (1000 per page)\n", - "Processing 936 of 1000 on page 2 - (1000 per page)\n", - "Processing 937 of 1000 on page 2 - (1000 per page)\n", - "Processing 938 of 1000 on page 2 - (1000 per page)\n", - "Processing 939 of 1000 on page 2 - (1000 per page)\n", - "Processing 940 of 1000 on page 2 - (1000 per page)\n", - "Processing 941 of 1000 on page 2 - (1000 per page)\n", - "Processing 942 of 1000 on page 2 - (1000 per page)\n", - "Processing 943 of 1000 on page 2 - (1000 per page)\n", - "Processing 944 of 1000 on page 2 - (1000 per page)\n", - "Processing 945 of 1000 on page 2 - (1000 per page)\n", - "Processing 946 of 1000 on page 2 - (1000 per page)\n", - "Processing 947 of 1000 on page 2 - (1000 per page)\n", - "Processing 948 of 1000 on page 2 - (1000 per page)\n", - "Processing 949 of 1000 on page 2 - (1000 per page)\n", - "Processing 950 of 1000 on page 2 - (1000 per page)\n", - "Processing 951 of 1000 on page 2 - (1000 per page)\n", - "Processing 952 of 1000 on page 2 - (1000 per page)\n", - "Processing 953 of 1000 on page 2 - (1000 per page)\n", - "Processing 954 of 1000 on page 2 - (1000 per page)\n", - "Processing 955 of 1000 on page 2 - (1000 per page)\n", - "Processing 956 of 1000 on page 2 - (1000 per page)\n", - "Processing 957 of 1000 on page 2 - (1000 per page)\n", - "Processing 958 of 1000 on page 2 - (1000 per page)\n", - "Processing 959 of 1000 on page 2 - (1000 per page)\n", - "Processing 960 of 1000 on page 2 - (1000 per page)\n", - "Processing 961 of 1000 on page 2 - (1000 per page)\n", - "Processing 962 of 1000 on page 2 - (1000 per page)\n", - "Processing 963 of 1000 on page 2 - (1000 per page)\n", - "Processing 964 of 1000 on page 2 - (1000 per page)\n", - "Processing 965 of 1000 on page 2 - (1000 per page)\n", - "Processing 966 of 1000 on page 2 - (1000 per page)\n", - "Processing 967 of 1000 on page 2 - (1000 per page)\n", - "Processing 968 of 1000 on page 2 - (1000 per page)\n", - "Processing 969 of 1000 on page 2 - (1000 per page)\n", - "Processing 970 of 1000 on page 2 - (1000 per page)\n", - "Processing 971 of 1000 on page 2 - (1000 per page)\n", - "Processing 972 of 1000 on page 2 - (1000 per page)\n", - "Processing 973 of 1000 on page 2 - (1000 per page)\n", - "Processing 974 of 1000 on page 2 - (1000 per page)\n", - "Processing 975 of 1000 on page 2 - (1000 per page)\n", - "Processing 976 of 1000 on page 2 - (1000 per page)\n", - "Processing 977 of 1000 on page 2 - (1000 per page)\n", - "Processing 978 of 1000 on page 2 - (1000 per page)\n", - "Processing 979 of 1000 on page 2 - (1000 per page)\n", - "Processing 980 of 1000 on page 2 - (1000 per page)\n", - "Processing 981 of 1000 on page 2 - (1000 per page)\n", - "Processing 982 of 1000 on page 2 - (1000 per page)\n", - "Processing 983 of 1000 on page 2 - (1000 per page)\n", - "Processing 984 of 1000 on page 2 - (1000 per page)\n", - "Processing 985 of 1000 on page 2 - (1000 per page)\n", - "Processing 986 of 1000 on page 2 - (1000 per page)\n", - "Processing 987 of 1000 on page 2 - (1000 per page)\n", - "Processing 988 of 1000 on page 2 - (1000 per page)\n", - "Processing 989 of 1000 on page 2 - (1000 per page)\n", - "Processing 990 of 1000 on page 2 - (1000 per page)\n", - "Processing 991 of 1000 on page 2 - (1000 per page)\n", - "Processing 992 of 1000 on page 2 - (1000 per page)\n", - "Processing 993 of 1000 on page 2 - (1000 per page)\n", - "Processing 994 of 1000 on page 2 - (1000 per page)\n", - "Processing 995 of 1000 on page 2 - (1000 per page)\n", - "Processing 996 of 1000 on page 2 - (1000 per page)\n", - "Processing 997 of 1000 on page 2 - (1000 per page)\n", - "Processing 998 of 1000 on page 2 - (1000 per page)\n", - "Processing 999 of 1000 on page 2 - (1000 per page)\n", - "Processing 1000 of 1000 on page 2 - (1000 per page)\n", - "Processing 1 of 153 on page 3 - (1000 per page)\n", - "Processing 2 of 153 on page 3 - (1000 per page)\n", - "Processing 3 of 153 on page 3 - (1000 per page)\n", - "Processing 4 of 153 on page 3 - (1000 per page)\n", - "Processing 5 of 153 on page 3 - (1000 per page)\n", - "Processing 6 of 153 on page 3 - (1000 per page)\n", - "Processing 7 of 153 on page 3 - (1000 per page)\n", - "Processing 8 of 153 on page 3 - (1000 per page)\n", - "Processing 9 of 153 on page 3 - (1000 per page)\n", - "Processing 10 of 153 on page 3 - (1000 per page)\n", - "Processing 11 of 153 on page 3 - (1000 per page)\n", - "Processing 12 of 153 on page 3 - (1000 per page)\n", - "Processing 13 of 153 on page 3 - (1000 per page)\n", - "Processing 14 of 153 on page 3 - (1000 per page)\n", - "Processing 15 of 153 on page 3 - (1000 per page)\n", - "Processing 16 of 153 on page 3 - (1000 per page)\n", - "Processing 17 of 153 on page 3 - (1000 per page)\n", - "Processing 18 of 153 on page 3 - (1000 per page)\n", - "Processing 19 of 153 on page 3 - (1000 per page)\n", - "Processing 20 of 153 on page 3 - (1000 per page)\n", - "Processing 21 of 153 on page 3 - (1000 per page)\n", - "Processing 22 of 153 on page 3 - (1000 per page)\n", - "Processing 23 of 153 on page 3 - (1000 per page)\n", - "Processing 24 of 153 on page 3 - (1000 per page)\n", - "Processing 25 of 153 on page 3 - (1000 per page)\n", - "Processing 26 of 153 on page 3 - (1000 per page)\n", - "Processing 27 of 153 on page 3 - (1000 per page)\n", - "Processing 28 of 153 on page 3 - (1000 per page)\n", - "Processing 29 of 153 on page 3 - (1000 per page)\n", - "Processing 30 of 153 on page 3 - (1000 per page)\n", - "Processing 31 of 153 on page 3 - (1000 per page)\n", - "Processing 32 of 153 on page 3 - (1000 per page)\n", - "Processing 33 of 153 on page 3 - (1000 per page)\n", - "Processing 34 of 153 on page 3 - (1000 per page)\n", - "Processing 35 of 153 on page 3 - (1000 per page)\n", - "Processing 36 of 153 on page 3 - (1000 per page)\n", - "Processing 37 of 153 on page 3 - (1000 per page)\n", - "Processing 38 of 153 on page 3 - (1000 per page)\n", - "Processing 39 of 153 on page 3 - (1000 per page)\n", - "Processing 40 of 153 on page 3 - (1000 per page)\n", - "Processing 41 of 153 on page 3 - (1000 per page)\n", - "Processing 42 of 153 on page 3 - (1000 per page)\n", - "Processing 43 of 153 on page 3 - (1000 per page)\n", - "Processing 44 of 153 on page 3 - (1000 per page)\n", - "Processing 45 of 153 on page 3 - (1000 per page)\n", - "Processing 46 of 153 on page 3 - (1000 per page)\n", - "Processing 47 of 153 on page 3 - (1000 per page)\n", - "Processing 48 of 153 on page 3 - (1000 per page)\n", - "Processing 49 of 153 on page 3 - (1000 per page)\n", - "Processing 50 of 153 on page 3 - (1000 per page)\n", - "Processing 51 of 153 on page 3 - (1000 per page)\n", - "Processing 52 of 153 on page 3 - (1000 per page)\n", - "Processing 53 of 153 on page 3 - (1000 per page)\n", - "Processing 54 of 153 on page 3 - (1000 per page)\n", - "Processing 55 of 153 on page 3 - (1000 per page)\n", - "Processing 56 of 153 on page 3 - (1000 per page)\n", - "Processing 57 of 153 on page 3 - (1000 per page)\n", - "Processing 58 of 153 on page 3 - (1000 per page)\n", - "Processing 59 of 153 on page 3 - (1000 per page)\n", - "Processing 60 of 153 on page 3 - (1000 per page)\n", - "Processing 61 of 153 on page 3 - (1000 per page)\n", - "Processing 62 of 153 on page 3 - (1000 per page)\n", - "Processing 63 of 153 on page 3 - (1000 per page)\n", - "Processing 64 of 153 on page 3 - (1000 per page)\n", - "Processing 65 of 153 on page 3 - (1000 per page)\n", - "Processing 66 of 153 on page 3 - (1000 per page)\n", - "Processing 67 of 153 on page 3 - (1000 per page)\n", - "Processing 68 of 153 on page 3 - (1000 per page)\n", - "Processing 69 of 153 on page 3 - (1000 per page)\n", - "Processing 70 of 153 on page 3 - (1000 per page)\n", - "Processing 71 of 153 on page 3 - (1000 per page)\n", - "Processing 72 of 153 on page 3 - (1000 per page)\n", - "Processing 73 of 153 on page 3 - (1000 per page)\n", - "Processing 74 of 153 on page 3 - (1000 per page)\n", - "Processing 75 of 153 on page 3 - (1000 per page)\n", - "Processing 76 of 153 on page 3 - (1000 per page)\n", - "Processing 77 of 153 on page 3 - (1000 per page)\n", - "Processing 78 of 153 on page 3 - (1000 per page)\n", - "Processing 79 of 153 on page 3 - (1000 per page)\n", - "Processing 80 of 153 on page 3 - (1000 per page)\n", - "Processing 81 of 153 on page 3 - (1000 per page)\n", - "Processing 82 of 153 on page 3 - (1000 per page)\n", - "Processing 83 of 153 on page 3 - (1000 per page)\n", - "Processing 84 of 153 on page 3 - (1000 per page)\n", - "Processing 85 of 153 on page 3 - (1000 per page)\n", - "Processing 86 of 153 on page 3 - (1000 per page)\n", - "Processing 87 of 153 on page 3 - (1000 per page)\n", - "Processing 88 of 153 on page 3 - (1000 per page)\n", - "Processing 89 of 153 on page 3 - (1000 per page)\n", - "Processing 90 of 153 on page 3 - (1000 per page)\n", - "Processing 91 of 153 on page 3 - (1000 per page)\n", - "Processing 92 of 153 on page 3 - (1000 per page)\n", - "Processing 93 of 153 on page 3 - (1000 per page)\n", - "Processing 94 of 153 on page 3 - (1000 per page)\n", - "Processing 95 of 153 on page 3 - (1000 per page)\n", - "Processing 96 of 153 on page 3 - (1000 per page)\n", - "Processing 97 of 153 on page 3 - (1000 per page)\n", - "Processing 98 of 153 on page 3 - (1000 per page)\n", - "Processing 99 of 153 on page 3 - (1000 per page)\n", - "Processing 100 of 153 on page 3 - (1000 per page)\n", - "Processing 101 of 153 on page 3 - (1000 per page)\n", - "Processing 102 of 153 on page 3 - (1000 per page)\n", - "Processing 103 of 153 on page 3 - (1000 per page)\n", - "Processing 104 of 153 on page 3 - (1000 per page)\n", - "Processing 105 of 153 on page 3 - (1000 per page)\n", - "Processing 106 of 153 on page 3 - (1000 per page)\n", - "Processing 107 of 153 on page 3 - (1000 per page)\n", - "Processing 108 of 153 on page 3 - (1000 per page)\n", - "Processing 109 of 153 on page 3 - (1000 per page)\n", - "Processing 110 of 153 on page 3 - (1000 per page)\n", - "Processing 111 of 153 on page 3 - (1000 per page)\n", - "Processing 112 of 153 on page 3 - (1000 per page)\n", - "Processing 113 of 153 on page 3 - (1000 per page)\n", - "Processing 114 of 153 on page 3 - (1000 per page)\n", - "Processing 115 of 153 on page 3 - (1000 per page)\n", - "Processing 116 of 153 on page 3 - (1000 per page)\n", - "Processing 117 of 153 on page 3 - (1000 per page)\n", - "Processing 118 of 153 on page 3 - (1000 per page)\n", - "Processing 119 of 153 on page 3 - (1000 per page)\n", - "Processing 120 of 153 on page 3 - (1000 per page)\n", - "Processing 121 of 153 on page 3 - (1000 per page)\n", - "Processing 122 of 153 on page 3 - (1000 per page)\n", - "Processing 123 of 153 on page 3 - (1000 per page)\n", - "Processing 124 of 153 on page 3 - (1000 per page)\n", - "Processing 125 of 153 on page 3 - (1000 per page)\n", - "Processing 126 of 153 on page 3 - (1000 per page)\n", - "Processing 127 of 153 on page 3 - (1000 per page)\n", - "Processing 128 of 153 on page 3 - (1000 per page)\n", - "Processing 129 of 153 on page 3 - (1000 per page)\n", - "Processing 130 of 153 on page 3 - (1000 per page)\n", - "Processing 131 of 153 on page 3 - (1000 per page)\n", - "Processing 132 of 153 on page 3 - (1000 per page)\n", - "Processing 133 of 153 on page 3 - (1000 per page)\n", - "Processing 134 of 153 on page 3 - (1000 per page)\n", - "Processing 135 of 153 on page 3 - (1000 per page)\n", - "Processing 136 of 153 on page 3 - (1000 per page)\n", - "Processing 137 of 153 on page 3 - (1000 per page)\n", - "Processing 138 of 153 on page 3 - (1000 per page)\n", - "Processing 139 of 153 on page 3 - (1000 per page)\n", - "Processing 140 of 153 on page 3 - (1000 per page)\n", - "Processing 141 of 153 on page 3 - (1000 per page)\n", - "Processing 142 of 153 on page 3 - (1000 per page)\n", - "Processing 143 of 153 on page 3 - (1000 per page)\n", - "Processing 144 of 153 on page 3 - (1000 per page)\n", - "Processing 145 of 153 on page 3 - (1000 per page)\n", - "Processing 146 of 153 on page 3 - (1000 per page)\n", - "Processing 147 of 153 on page 3 - (1000 per page)\n", - "Processing 148 of 153 on page 3 - (1000 per page)\n", - "Processing 149 of 153 on page 3 - (1000 per page)\n", - "Processing 150 of 153 on page 3 - (1000 per page)\n", - "Processing 151 of 153 on page 3 - (1000 per page)\n", - "Processing 152 of 153 on page 3 - (1000 per page)\n", - "Processing 153 of 153 on page 3 - (1000 per page)\n", - "usgs_elev_tables load completed\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Has appx 2,150 HUCs to process, but this section goes quickly.\n", @@ -3247,15 +1010,15 @@ " 'Delimiter': '/'}\n", "page_iterator = paginator.paginate(**operation_parameters)\n", "page_count = 0\n", - "# num_pages = len(page_iterator) pagintors don't have a length parameter\n", - "#print(f\"num of pages is {num_pages}\")\n", + "num_pages = paginator.num_pages\n", + "\n", "for page in page_iterator:\n", "\n", " prefix_objects = page['CommonPrefixes']\n", " for i, prefix_obj in enumerate(prefix_objects):\n", - " display_index = (i + 1) + (1000 * page_count)\n", - " print(f\"Processing {i+1} of\"\n", - " f\" {len(prefix_objects)} on page {page_count + 1} - (1000 per page)\")\n", + " rec_num = (i + 1) + (1000 * page_count)\n", + " print(f\"Processing rec number {rec_num} : \"\n", + " f\" On page {page_count} of {num_pages} (up to 1000 per page)\")\n", " huc_prefix = prefix_obj.get(\"Prefix\")\n", " usgs_elev_table_key = f'{huc_prefix}usgs_elev_table.csv'\n", " try:\n", @@ -3303,186 +1066,344 @@ "metadata": { "tags": [] }, + "outputs": [], + "source": [ + "\n", + "# Takes appx 5.75 to 6 hrs to run\n", + "\n", + "# ****** WIP TESTS to see if we can speed it up ****\n", + "\n", + "# Can not use Multi proc as there can be collisions to the db.\n", + "\n", + "# print(\"hydrotable reloaded - started\")\n", + "# start_dt = datetime.now()\n", + "\n", + "# sf.execute_sql('DROP TABLE IF EXISTS derived.hydrotable;')\n", + "# sql = '''\n", + "# SELECT distinct LPAD(huc8::text, 8, '0') as huc8 FROM derived.featureid_huc_crosswalk WHERE huc8 is not null;\n", + "# '''\n", + "# df = sf.sql_to_dataframe(sql)\n", + "# ht_usecols = ['HydroID', 'feature_id', 'stage', 'discharge_cms']\n", + "\n", + "# paginator = S3_CLIENT.get_paginator('list_objects')\n", + "# operation_parameters = {'Bucket': FIM_BUCKET,\n", + "# 'Prefix': f'{HAND_DATASETS_DPATH}/',\n", + "# 'Delimiter': '/'}\n", + "# page_iterator = paginator.paginate(**operation_parameters)\n", + "\n", + "# # print(len(page_iterator.items))\n", + "\n", + "# page_count = 0\n", + "\n", + "# # Incomplete tests for improving performance\n", + "# number_of_recs_test = 50\n", + "# test_done = False\n", + "# df = pd.DataFrame()\n", + "# is_last_pg_chunck = False\n", + "# save_chunk_to_db = False\n", + "# num_files_per_df = 20\n", + "\n", + "# for page in page_iterator:\n", + "\n", + "# if test_done:\n", + "# break\n", + " \n", + "# prefix_objects = page['CommonPrefixes']\n", + "# # num_of_rec_in_pg = len(prefix_objects)\n", + "# # print(f\"num of records in page is {num_of_rec_in_pg}\")\n", + "\n", + "# for i, prefix_obj in enumerate(prefix_objects):\n", + "\n", + " \n", + "# if (i + 1) > number_of_recs_test:\n", + "# print(\"test done\")\n", + "# test_done = True\n", + "# break\n", + " \n", + "# total_recs_in_page = len(prefix_obj)\n", + "# print(f\"Total recs in set is {total_recs_in_page}\")\n", + " \n", + "# # test override for total recs\n", + "# total_recs_in_page = 50\n", + " \n", + "# rec_num = (i + 1) + (1000 * page_count)\n", + "# print(f\"Processing rec number {rec_num} : \"\n", + "# f\" On page {page_count + 1} of (number of pages unknown) - (up to 1000 per page)\")\n", + "\n", + "# branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", + "# branch_files_result = S3_CLIENT.list_objects(\n", + "# Bucket=FIM_BUCKET, \n", + "# Prefix=branch_prefix, \n", + "# Delimiter='/'\n", + "# )\n", + "# hydro_table_key = None\n", + "# for content_obj in branch_files_result.get('Contents'):\n", + "# branch_file_prefix = content_obj['Key']\n", + "# if 'hydroTable' in branch_file_prefix:\n", + "# hydro_table_key = branch_file_prefix\n", + "\n", + "# if hydro_table_key:\n", + " \n", + "# # print(f\"Found usgs_elev_table and hydroTable in {branch_prefix}\")\n", + "# try:\n", + "# # print(\"...Fetching csvs...\")\n", + "# ht = S3_CLIENT.get_object(\n", + "# Bucket=FIM_BUCKET,\n", + "# Key=hydro_table_key\n", + "# )['Body']\n", + "# # print(\"...Reading with pandas...\")\n", + "# ht_df = pd.read_csv(ht, header=0, usecols=ht_usecols)\n", + " \n", + "# mod_num = (i + 1) % num_files_per_df\n", + "# print(\"========================\")\n", + "# print(f\"mod is {mod_num}\")\n", + "# print(f\"+++ current length of ht_df is {len(ht_df)}\")\n", + "# print(f\"+++ current length of df is {len(df)}\")\n", + "\n", + "# # No matter what, append to the df (or start a new one if length is zero)\n", + "# # which is likely a new set\n", + "# if len(df) == 0:\n", + "# df = ht_df # sets up the schema\n", + "# else:\n", + "# df = pd.concat([df, ht_df])\n", + "# print(\"... concating db's\")\n", + " \n", + "# print(f\"++++ new length of df is {len(df)}\")\n", + " \n", + "# if mod_num == 0:\n", + "# print(\" ***** Saving df to database\")\n", + "# save_chunk_to_db = True\n", + "# elif (i + 1 >= total_recs_in_page):\n", + "# print(\"*** last set of recs in this page set, saving df to database\")\n", + "# save_chunk_to_db = True\n", + "# else:\n", + "# # not saving to Db yet, just build up larger sets with 20 csv's in them \n", + "# # before loading to db.\n", + "# continue # not one of the sets of twenty\n", + "\n", + "# # if we got here, just save the db\n", + "# print(\"\")\n", + "# print(\".... saving dataframe to db\")\n", + "# df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", + "# df.to_sql(\n", + "# con=VIZ_DB_ENGINE, \n", + "# schema='derived',\n", + "# name='hydrotable',\n", + "# index=False,\n", + "# if_exists='append',\n", + "# method='multi'\n", + "# )\n", + "# save_chunk_to_db = False\n", + "# df = pd.DataFrame()\n", + "\n", + "# except Exception as e:\n", + "# raise e\n", + "# print(f'Fetch failed: {e}')\n", + "\n", + "# page_count += 1\n", + "\n", + "# end_dt = datetime.now()\n", + "# time_duration = end_dt - start_dt\n", + "# print(\"hydrotable reload done\")\n", + "# print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", + "\n", + "# '''\n", + "# Timing tests..\n", + "# length of 50 (groups of 20)\n", + "# - one rec downloaded and to_sql at a time = 5:40\n", + "# - Group into sets of 20 recs, then send to DF, make sure we don't miss the last one. = still 5:36\n", + "# - add the method=\"multi\" arg = same and tended to freeze\n", + "# '''\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "329dae61-d69f-4865-bdb6-e99a7442b0be", + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "hydrotable reloaded - started\n", - "Processing 1 of 1000 on page 1 - (1000 per page)\n", - "Processing 2 of 1000 on page 2 - (1000 per page)\n", - "Processing 3 of 1000 on page 3 - (1000 per page)\n", - "Processing 4 of 1000 on page 4 - (1000 per page)\n", - "Processing 5 of 1000 on page 5 - (1000 per page)\n", - "Processing 6 of 1000 on page 6 - (1000 per page)\n", - "Processing 7 of 1000 on page 7 - (1000 per page)\n", - "Processing 8 of 1000 on page 8 - (1000 per page)\n", - "Processing 9 of 1000 on page 9 - (1000 per page)\n", - "Processing 10 of 1000 on page 10 - (1000 per page)\n", - "Processing 11 of 1000 on page 11 - (1000 per page)\n", - "Processing 12 of 1000 on page 12 - (1000 per page)\n", - "Processing 13 of 1000 on page 13 - (1000 per page)\n", - "Processing 14 of 1000 on page 14 - (1000 per page)\n", - "Processing 15 of 1000 on page 15 - (1000 per page)\n", - "Processing 16 of 1000 on page 16 - (1000 per page)\n", - "Processing 17 of 1000 on page 17 - (1000 per page)\n", - "Processing 18 of 1000 on page 18 - (1000 per page)\n", - "Processing 19 of 1000 on page 19 - (1000 per page)\n", - "Processing 20 of 1000 on page 20 - (1000 per page)\n", - "Processing 21 of 1000 on page 21 - (1000 per page)\n", - "Processing 22 of 1000 on page 22 - (1000 per page)\n", - "Processing 23 of 1000 on page 23 - (1000 per page)\n", - "Processing 24 of 1000 on page 24 - (1000 per page)\n", - "Processing 25 of 1000 on page 25 - (1000 per page)\n", - "Processing 26 of 1000 on page 26 - (1000 per page)\n", - "Processing 27 of 1000 on page 27 - (1000 per page)\n", - "Processing 28 of 1000 on page 28 - (1000 per page)\n", - "Processing 29 of 1000 on page 29 - (1000 per page)\n", - "Processing 30 of 1000 on page 30 - (1000 per page)\n", - "Processing 31 of 1000 on page 31 - (1000 per page)\n", - "Processing 32 of 1000 on page 32 - (1000 per page)\n", - "Processing 33 of 1000 on page 33 - (1000 per page)\n", - "Processing 34 of 1000 on page 34 - (1000 per page)\n", - "Processing 35 of 1000 on page 35 - (1000 per page)\n", - "Processing 36 of 1000 on page 36 - (1000 per page)\n", - "Processing 37 of 1000 on page 37 - (1000 per page)\n", - "Processing 38 of 1000 on page 38 - (1000 per page)\n", - "Processing 39 of 1000 on page 39 - (1000 per page)\n", - "Processing 40 of 1000 on page 40 - (1000 per page)\n", - "Processing 41 of 1000 on page 41 - (1000 per page)\n", - "Processing 42 of 1000 on page 42 - (1000 per page)\n", - "Processing 43 of 1000 on page 43 - (1000 per page)\n", - "Processing 44 of 1000 on page 44 - (1000 per page)\n", - "Processing 45 of 1000 on page 45 - (1000 per page)\n", - "Processing 46 of 1000 on page 46 - (1000 per page)\n", - "Processing 47 of 1000 on page 47 - (1000 per page)\n", - "Processing 48 of 1000 on page 48 - (1000 per page)\n", - "Processing 49 of 1000 on page 49 - (1000 per page)\n", - "Processing 50 of 1000 on page 50 - (1000 per page)\n", - "Processing 51 of 1000 on page 51 - (1000 per page)\n", - "Processing 52 of 1000 on page 52 - (1000 per page)\n", - "Processing 53 of 1000 on page 53 - (1000 per page)\n", - "Processing 54 of 1000 on page 54 - (1000 per page)\n", - "Processing 55 of 1000 on page 55 - (1000 per page)\n", - "Processing 56 of 1000 on page 56 - (1000 per page)\n", - "Processing 57 of 1000 on page 57 - (1000 per page)\n", - "Processing 58 of 1000 on page 58 - (1000 per page)\n", - "Processing 59 of 1000 on page 59 - (1000 per page)\n", - "Processing 60 of 1000 on page 60 - (1000 per page)\n", - "Processing 61 of 1000 on page 61 - (1000 per page)\n", - "Processing 62 of 1000 on page 62 - (1000 per page)\n", - "Processing 63 of 1000 on page 63 - (1000 per page)\n", - "Processing 64 of 1000 on page 64 - (1000 per page)\n", - "Processing 65 of 1000 on page 65 - (1000 per page)\n", - "Processing 66 of 1000 on page 66 - (1000 per page)\n", - "Processing 67 of 1000 on page 67 - (1000 per page)\n", - "Processing 68 of 1000 on page 68 - (1000 per page)\n", - "Processing 69 of 1000 on page 69 - (1000 per page)\n", - "Processing 70 of 1000 on page 70 - (1000 per page)\n", - "Processing 71 of 1000 on page 71 - (1000 per page)\n", - "Processing 72 of 1000 on page 72 - (1000 per page)\n", - "Processing 73 of 1000 on page 73 - (1000 per page)\n", - "Processing 74 of 1000 on page 74 - (1000 per page)\n", - "Processing 75 of 1000 on page 75 - (1000 per page)\n", - "Processing 76 of 1000 on page 76 - (1000 per page)\n", - "Processing 77 of 1000 on page 77 - (1000 per page)\n", - "Processing 78 of 1000 on page 78 - (1000 per page)\n", - "Processing 79 of 1000 on page 79 - (1000 per page)\n", - "Processing 80 of 1000 on page 80 - (1000 per page)\n", - "Processing 81 of 1000 on page 81 - (1000 per page)\n", - "Processing 82 of 1000 on page 82 - (1000 per page)\n", - "Processing 83 of 1000 on page 83 - (1000 per page)\n", - "Processing 84 of 1000 on page 84 - (1000 per page)\n", - "Processing 85 of 1000 on page 85 - (1000 per page)\n", - "Processing 86 of 1000 on page 86 - (1000 per page)\n", - "Processing 87 of 1000 on page 87 - (1000 per page)\n", - "Processing 88 of 1000 on page 88 - (1000 per page)\n", - "Processing 89 of 1000 on page 89 - (1000 per page)\n", - "Processing 90 of 1000 on page 90 - (1000 per page)\n", - "Processing 91 of 1000 on page 91 - (1000 per page)\n", - "Processing 92 of 1000 on page 92 - (1000 per page)\n", - "Processing 93 of 1000 on page 93 - (1000 per page)\n", - "Processing 94 of 1000 on page 94 - (1000 per page)\n", - "Processing 95 of 1000 on page 95 - (1000 per page)\n", - "Processing 96 of 1000 on page 96 - (1000 per page)\n", - "Processing 97 of 1000 on page 97 - (1000 per page)\n", - "Processing 98 of 1000 on page 98 - (1000 per page)\n", - "Processing 99 of 1000 on page 99 - (1000 per page)\n", - "Processing 100 of 1000 on page 100 - (1000 per page)\n", - "Processing 101 of 1000 on page 101 - (1000 per page)\n", - "Processing 102 of 1000 on page 102 - (1000 per page)\n", - "Processing 103 of 1000 on page 103 - (1000 per page)\n", - "Processing 104 of 1000 on page 104 - (1000 per page)\n", - "Processing 105 of 1000 on page 105 - (1000 per page)\n", - "Processing 106 of 1000 on page 106 - (1000 per page)\n", - "Processing 107 of 1000 on page 107 - (1000 per page)\n", - "Processing 108 of 1000 on page 108 - (1000 per page)\n", - "Processing 109 of 1000 on page 109 - (1000 per page)\n", - "Processing 110 of 1000 on page 110 - (1000 per page)\n", - "Processing 111 of 1000 on page 111 - (1000 per page)\n", - "Processing 112 of 1000 on page 112 - (1000 per page)\n", - "Processing 113 of 1000 on page 113 - (1000 per page)\n", - "Processing 114 of 1000 on page 114 - (1000 per page)\n", - "Processing 115 of 1000 on page 115 - (1000 per page)\n", - "Processing 116 of 1000 on page 116 - (1000 per page)\n", - "Processing 117 of 1000 on page 117 - (1000 per page)\n", - "Processing 118 of 1000 on page 118 - (1000 per page)\n", - "Processing 119 of 1000 on page 119 - (1000 per page)\n", - "Processing 120 of 1000 on page 120 - (1000 per page)\n", - "Processing 121 of 1000 on page 121 - (1000 per page)\n", - "Processing 122 of 1000 on page 122 - (1000 per page)\n", - "Processing 123 of 1000 on page 123 - (1000 per page)\n", - "Processing 124 of 1000 on page 124 - (1000 per page)\n", - "Processing 125 of 1000 on page 125 - (1000 per page)\n", - "Processing 126 of 1000 on page 126 - (1000 per page)\n", - "Processing 127 of 1000 on page 127 - (1000 per page)\n", - "Processing 128 of 1000 on page 128 - (1000 per page)\n", - "Processing 129 of 1000 on page 129 - (1000 per page)\n", - "Processing 130 of 1000 on page 130 - (1000 per page)\n", - "Processing 131 of 1000 on page 131 - (1000 per page)\n", - "Processing 132 of 1000 on page 132 - (1000 per page)\n", - "Processing 133 of 1000 on page 133 - (1000 per page)\n", - "Processing 134 of 1000 on page 134 - (1000 per page)\n", - "Processing 135 of 1000 on page 135 - (1000 per page)\n", - "Processing 136 of 1000 on page 136 - (1000 per page)\n", - "Processing 137 of 1000 on page 137 - (1000 per page)\n", - "Processing 138 of 1000 on page 138 - (1000 per page)\n", - "Processing 139 of 1000 on page 139 - (1000 per page)\n", - "Processing 140 of 1000 on page 140 - (1000 per page)\n", - "Processing 141 of 1000 on page 141 - (1000 per page)\n", - "Processing 142 of 1000 on page 142 - (1000 per page)\n", - "Processing 143 of 1000 on page 143 - (1000 per page)\n", - "Processing 144 of 1000 on page 144 - (1000 per page)\n", - "Processing 145 of 1000 on page 145 - (1000 per page)\n", - "Processing 146 of 1000 on page 146 - (1000 per page)\n", - "Processing 147 of 1000 on page 147 - (1000 per page)\n", - "Processing 148 of 1000 on page 148 - (1000 per page)\n", - "Processing 149 of 1000 on page 149 - (1000 per page)\n", - "Processing 150 of 1000 on page 150 - (1000 per page)\n", - "Processing 151 of 1000 on page 151 - (1000 per page)\n", - "Processing 152 of 1000 on page 152 - (1000 per page)\n", - "Processing 153 of 1000 on page 153 - (1000 per page)\n", - "Processing 154 of 1000 on page 154 - (1000 per page)\n", - "Processing 155 of 1000 on page 155 - (1000 per page)\n", - "Processing 156 of 1000 on page 156 - (1000 per page)\n", - "Processing 157 of 1000 on page 157 - (1000 per page)\n", - "Processing 158 of 1000 on page 158 - (1000 per page)\n", - "Processing 159 of 1000 on page 159 - (1000 per page)\n", - "Processing 160 of 1000 on page 160 - (1000 per page)\n", - "Processing 161 of 1000 on page 161 - (1000 per page)\n", - "Processing 162 of 1000 on page 162 - (1000 per page)\n", - "Processing 163 of 1000 on page 163 - (1000 per page)\n", - "Processing 164 of 1000 on page 164 - (1000 per page)\n", - "Processing 165 of 1000 on page 165 - (1000 per page)\n", - "Processing 166 of 1000 on page 166 - (1000 per page)\n", - "Processing 167 of 1000 on page 167 - (1000 per page)\n", - "Processing 168 of 1000 on page 168 - (1000 per page)\n" + "Processing rec number 1 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 2 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 3 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 4 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 5 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 6 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 7 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 8 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 9 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 10 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 11 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 12 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 13 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 14 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 15 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 16 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 17 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 18 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 19 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 20 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 21 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 22 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 23 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 24 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 25 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 26 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 27 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 28 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 29 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 30 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 31 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 32 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 33 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 34 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 35 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 36 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 37 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 38 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 39 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 40 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 41 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 42 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 43 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 44 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 45 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 46 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 47 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 48 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 49 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 50 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 51 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 52 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 53 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 54 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 55 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 56 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 57 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 58 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 59 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 60 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 61 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 62 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 63 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 64 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 65 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 66 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 67 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 68 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 69 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 70 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 71 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 72 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 73 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 74 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 75 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 76 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 77 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 78 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 79 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 80 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 81 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 82 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 83 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 84 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 85 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 86 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 87 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 88 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 89 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 90 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 91 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 92 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 93 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 94 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 95 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 96 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 97 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 98 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 99 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 100 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 101 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 102 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 103 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 104 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 105 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 106 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 107 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 108 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 109 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 110 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 111 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 112 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 113 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 114 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 115 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 116 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 117 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 118 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 119 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 120 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 121 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 122 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 123 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 124 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 125 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 126 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 127 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 128 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 129 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 130 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 131 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 132 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 133 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 134 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 135 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 136 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 137 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 138 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 139 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 140 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 141 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 142 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 143 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 144 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 145 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 146 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 147 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 148 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 149 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 150 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 151 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 152 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 153 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 154 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 155 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 156 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 157 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 158 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 159 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 160 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 161 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 162 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 163 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 164 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 165 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 166 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 167 : On page 1 (appx 2150 recs in total)\n", + "Processing rec number 168 : On page 1 (appx 2150 recs in total)\n" ] } ], "source": [ "\n", - "# Takes appx 5.75 to 6 hrs to run\n", + "# Note: This version finished in 1hr 49m but had to be babysat, clicking on the screen very 10 - 15 mins\n", "\n", "print(\"hydrotable reloaded - started\")\n", "start_dt = datetime.now()\n", @@ -3499,14 +1420,24 @@ " 'Prefix': f'{HAND_DATASETS_DPATH}/',\n", " 'Delimiter': '/'}\n", "page_iterator = paginator.paginate(**operation_parameters)\n", + "\n", + "# print(len(page_iterator.items))\n", "page_count = 0\n", + "\n", "for page in page_iterator:\n", "\n", " prefix_objects = page['CommonPrefixes']\n", + " # num_of_rec_in_pg = len(prefix_objects)\n", + " # print(f\"num of records in page is {num_of_rec_in_pg}\")\n", + "\n", " for i, prefix_obj in enumerate(prefix_objects):\n", "\n", - " print(f\"Processing {i+1} of\"\n", - " f\" {len(prefix_objects)} on page {page_count + 1} - (1000 per page)\")\n", + " # print(\"========================\")\n", + " total_recs_in_page = len(prefix_objects)\n", + "\n", + " rec_num = (i + 1) + (1000 * page_count)\n", + " print(f\"Processing rec number {rec_num} : \"\n", + " f\" On page {page_count + 1} (appx 2150 recs in total)\")\n", "\n", " branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", " branch_files_result = S3_CLIENT.list_objects(\n", @@ -3521,6 +1452,7 @@ " hydro_table_key = branch_file_prefix\n", "\n", " if hydro_table_key:\n", + "\n", " # print(f\"Found usgs_elev_table and hydroTable in {branch_prefix}\")\n", " try:\n", " # print(\"...Fetching csvs...\")\n", @@ -3528,28 +1460,33 @@ " Bucket=FIM_BUCKET,\n", " Key=hydro_table_key\n", " )['Body']\n", - " # print(\"...Reading with pandas...\")\n", + " # print(\"... Reading with pandas ...\")\n", " ht_df = pd.read_csv(ht, header=0, usecols=ht_usecols)\n", - " # print('...Writing to db...')\n", - " # ht_df['fim_version'] = PUBLIC_FIM_VERSION\n", + "\n", + " # print(\"... saving dataframe to db\")\n", " ht_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", " ht_df.to_sql(\n", - " con=VIZ_DB_ENGINE, \n", + " con=VIZ_DB_ENGINE,\n", " schema='derived',\n", " name='hydrotable',\n", " index=False,\n", - " if_exists='append'\n", + " if_exists='append',\n", + " method='multi'\n", " )\n", + " # print(\"... saved to db\")\n", + "\n", " except Exception as e:\n", " raise e\n", " print(f'Fetch failed: {e}')\n", "\n", - " page_count += 1\n", + " page_count += 1\n", "\n", "end_dt = datetime.now()\n", "time_duration = end_dt - start_dt\n", + "print(\"***************\")\n", "print(\"hydrotable reload done\")\n", - "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", + "print(\"\")\n" ] }, { @@ -3741,94 +1678,11 @@ { "cell_type": "code", "execution_count": null, - "id": "95c472f8-0557-4864-8536-1814f3ac4286", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Already run for 4.4.0.0 (4.5.2.11)\n", - "\n", - "'''\n", - "Be Very Careful to just rename tables. If they have indexes, the index will now point to the new\n", - "table names but maintain the original index name. Those index names can really mess stuff up.\n", - "Best to never rename unless you rename indexes as well. This particular on is ok. \n", - "Note: When various '\"to_sql\" tools are run which have GIST indexes, this index column name issue\n", - "will be the problem.\n", - "\n", - "Why Drop instead of Truncate? if the schema changes for the incoming, truncate will have column\n", - "missmatches.\n", - "\n", - "We really should be backing up indexes and constraints as well.\n", - "\n", - "'''\n", - "\n", - "# TODO: Aug 2024: Change this away from \"rename\" to copy / drop. \n", - "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.src_skill_temp RENAME TO src_skill_temp_{OLD_FIM_TAG};')\n", - "# sf.execute_sql(f'ALTER TABLE IF EXISTS derived.src_skill RENAME TO src_skill_{OLD_FIM_TAG};')\n", - "\n", - "# print(\"src_skill and src_skill_temps db renamed\")\n", - "\n", - "\n", - "# TODO: Rob Aug 2024: change this to backup of table and not rename as it messses with indexes\n", - "# Don't need a copy of the reference src_skill table , so just drop it.\n", - "new_table_name = f\"derived.src_skill_temp_{OLD_FIM_TAG}\"\n", - "sql = f'''\n", - " CREATE TABLE IF NOT EXISTS {new_table_name} AS TABLE derived.src_skill_temp;\n", - "'''\n", - "\n", - "\n", - "#print(\"src_skill and src_skill_temps db renamed\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "id": "9a249854-b9cd-4ccb-afea-96c35708515d", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done dropping src_skill and src_skill_temp tables\n" - ] - } - ], - "source": [ - "# Prep the dbs for the new load\n", - "#sf.eecute_sql('DROP TABLE IF EXISTS derived.src_skill_temp;', db_type='viz')\n", - "#print(\"Done dropping src_skill and src_skill_temp tables\")" - ] - }, - { - "cell_type": "code", - "execution_count": 53, "id": "c24f509a-90df-492a-a0d4-aa9eab8e7041", "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Reading file...\n", - ".. Downloading: s3://hydrovis-ti-deployment-us-east-1/fim/hand_4_5_11_1/qa_datasets/agg_nwm_recurr_flow_elev_stats_location_id.csv\n", - "\n", - ".. 1 files downloaded and loaded into the dataframe\n", - "File read. 2967 records found\n", - "Done loading derived.src_skill_temp table\n", - "... duration was 0:00:00\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Load the src_skill_temp table\n", @@ -3863,21 +1717,12 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "id": "b5639447-9a5a-4add-9c36-ebd945a5a8d0", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done loading derived.src_skill table\n", - "... duration was 0:00:00\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Load into src_skill table adding geometry to it from external.usgs_gage. Yes.. more/less straight from WRDS tables\n", @@ -3924,21 +1769,12 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "id": "0df23345-47a4-490a-b10f-2020e3a2a68e", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully copied derived.src_skill from the VIZ DB to reference.src_skill in the EGIS DB!\n", - "Done\n" - ] - } - ], + "outputs": [], "source": [ "\n", "sf.move_data_from_viz_to_egis(\"derived.src_skill\", \"reference.src_skill\")\n", @@ -3957,30 +1793,18 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "id": "f4fcdfab-3936-4a0a-8ab9-605e9a64f9bd", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, - "outputs": [ - { - "ename": "SyntaxError", - "evalue": "syntax error at or near \"EXISTS\"\nLINE 1: TRUNCATE TABLE IF EXISTS reference.fim_performance_points\n ^\n", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mSyntaxError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[58], line 14\u001b[0m\n\u001b[1;32m 12\u001b[0m sql \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTRUNCATE TABLE IF EXISTS \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtb_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;66;03m# print(sql)\u001b[39;00m\n\u001b[0;32m---> 14\u001b[0m \u001b[43msf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute_sql\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m,\u001b[49m\u001b[43mdb_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43megis\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAll fim_performance tables trunated if they exist\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/SageMaker/rob.hanna/code/ti_add_upload_script_hand_4_5_11_1/Core/Manual_Workflows/FIM_data_loads/../helper_functions/shared_functions.py:215\u001b[0m, in \u001b[0;36mexecute_sql\u001b[0;34m(sql, db_type)\u001b[0m\n\u001b[1;32m 213\u001b[0m db_connection\u001b[38;5;241m.\u001b[39mcommit()\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 215\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 217\u001b[0m db_connection\u001b[38;5;241m.\u001b[39mclose()\n", - "File \u001b[0;32m~/SageMaker/rob.hanna/code/ti_add_upload_script_hand_4_5_11_1/Core/Manual_Workflows/FIM_data_loads/../helper_functions/shared_functions.py:212\u001b[0m, in \u001b[0;36mexecute_sql\u001b[0;34m(sql, db_type)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 211\u001b[0m cur \u001b[38;5;241m=\u001b[39m db_connection\u001b[38;5;241m.\u001b[39mcursor()\n\u001b[0;32m--> 212\u001b[0m \u001b[43mcur\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 213\u001b[0m db_connection\u001b[38;5;241m.\u001b[39mcommit()\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", - "\u001b[0;31mSyntaxError\u001b[0m: syntax error at or near \"EXISTS\"\nLINE 1: TRUNCATE TABLE IF EXISTS reference.fim_performance_points\n ^\n" - ] - } - ], + "outputs": [], "source": [ "# clean up tables for new load\n", "\n", - "\n", "table_names = [\n", " \"reference.fim_performance_points\",\n", " \"reference.fim_performance_polys\",\n", @@ -4007,9 +1831,9 @@ "outputs": [], "source": [ "\n", - "# Load the new fim performance tables\n", + "# Load the new fim performance tables \n", "\n", - "start_dt = datetime.now()\n", + "start_dt = datetime.now() # appx 35 min\n", "\n", "# os.environ['EGIS_DB_HOST'] ='' #TI DB\n", "\n", @@ -4021,26 +1845,31 @@ "bucket = \"hydrovis-ti-deployment-us-east-1\"\n", "\n", "# file_handles = ['fim_performance_points.csv']\n", - "# file_handles = ['fim_performance_points.csv', 'fim_performance_polys.csv', 'fim_performance_catchments_dissolved.csv']\n", + "#file_handles = ['fim_performance_points.csv', 'fim_performance_polys.csv', 'fim_performance_catchments.csv']\n", "# file_handles = ['fim_performance_points.csv', 'fim_performance_polys.csv']\n", "file_handles = ['fim_performance_catchments.csv']\n", "\n", "for file_handle in file_handles:\n", "\n", - " print(\"Reading file...\")\n", " # df = pd.read_csv(local_download_path)\n", + " # print(f\"Reading file {file_to_download}\")\n", + " print(\"\")\n", " file_to_download = f\"{QA_DATASETS_DPATH}/{file_handle}\"\n", + " # ie) /fim/hand_4_5_11_1/qa_datasets/fim_performance_catchments.csv\n", " df = s3_sf.download_S3_csv_files_to_df_from_list(FIM_BUCKET, [file_to_download], True)\n", - " print(\"File read.\")\n", + " print(\"... File read.\")\n", "\n", - " # Rename headers.\n", + " # drop it for now, we will rebuild it\n", + " if 'oid' in df.columns:\n", + " df = df.drop('oid', axis=1)\n", "\n", + " # Rename headers.\n", " if file_handle == 'fim_performance_points.csv':\n", " df = df.rename(columns={'Unnamed: 0': 'oid', 'geometry': 'geom'})\n", " else:\n", " df = df.rename(columns={'Unnamed: 0': 'oid', 'geometry': 'geom', 'huc':'huc8'})\n", "\n", - " print(df.dtypes)\n", + " # print(df.dtypes)\n", " # Convert all field names to lowercase (needed for ArcGIS Pro).\n", " df.columns = df.columns.str.lower()\n", "\n", @@ -4073,19 +1902,22 @@ " else:\n", " df['huc8'] = df['huc8'].apply(lambda x: x.zfill(8))\n", "\n", - " df['version'] = PUBLIC_FIM_VERSION\n", - " df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", + " # add model version column\n", + " df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", + "\n", + " if \"version\" in df:\n", + " df = df.drop(\"version\", axis=1)\n", "\n", " # Upload df to database.\n", " stripped_layer_name = file_handle.replace(\".csv\", \"\")\n", " table_name = \"reference.\" + stripped_layer_name\n", - " print(\"Loading data into DB...\")\n", + " print(\"... Loading data into DB\")\n", "\n", " # Chunk load data into DB\n", "\n", " if file_handle in ['fim_performance_catchments.csv']:\n", "\n", - " print(\"Chunk loading...\")\n", + " print(\"... Chunk loading\")\n", " # Create list of df chunks\n", " n = 10000 # chunk row size\n", " list_df = [df[i:i+n] for i in range(0,df.shape[0],n)]\n", @@ -4095,10 +1927,10 @@ " print(first_chunk_df.shape[0])\n", "\n", " first_chunk_df.to_sql(\n", - " name=stripped_layer_name, \n", - " con=db_engine, \n", + " name=stripped_layer_name,\n", + " con=db_engine,\n", " schema='reference',\n", - " if_exists='replace', \n", + " if_exists='replace',\n", " index=False,\n", " dtype={'oid': sqlalchemy.types.Integer(),\n", " 'version': sqlalchemy.types.String(),\n", @@ -4121,8 +1953,8 @@ " }\n", " )\n", " else:\n", - " if 'points' in stripped_layer_name: geometry = 'POINT'\n", - " if 'polys' in stripped_layer_name: geometry = 'POLYGON'\n", + " if 'points' in stripped_layer_name: geometry='POINT'\n", + " if 'polys' in stripped_layer_name: geometry='POLYGON'\n", " # print(\"GEOMETRY\")\n", " # print(geometry)\n", " df.to_sql(\n", @@ -4137,7 +1969,8 @@ " }\n", " )\n", "\n", - " print(f\">>> {file_handle} downloaded and loaded\")\n", + " print(f\"... >>> {file_handle} downloaded and loaded\")\n", + " print(\"\")\n", "\n", " # deleted the downloaded file that was just processed.\n", " # if os.path.exists(local_download_path):\n", diff --git a/Core/Manual_Workflows/helper_functions/s3_shared_functions.py b/Core/Manual_Workflows/helper_functions/s3_shared_functions.py index 8a7a41e7..9cd99d06 100644 --- a/Core/Manual_Workflows/helper_functions/s3_shared_functions.py +++ b/Core/Manual_Workflows/helper_functions/s3_shared_functions.py @@ -100,43 +100,39 @@ # ***************************** # CAUTION: TODO: Aug 2024: This needs to be re-thought. I can easily overpower the notebook server depending on teh size of the notebooks # ***************************** -def download_S3_csv_files_to_df_from_list(bucket_name, list_file_paths, is_verbose=False): +def load_S3_csv_to_df(s3_file_path, is_verbose=False): ''' Overview: - - Send a list of file paths (fully qualified, except for the "s3://" and bucket name" + - a full s3 file path + ie) s3://hydrovis-ti-deployment-us-east-1/fim/hand_4_5_11_1/qa_datasets/fim_performance_catchments.csv Returns: - - A dataframe with the + - A dataframe with the csv data loaded ''' - if len(list_file_paths) == 0: - raise Exception("No files requested for download") - - rtn_df = None - + if s3_file_path is None or s3_file_path == '' or len(s3_file_path) < 5: + raise Exception("s3_file_path is invalid - not set, empty or too short") + + if s3_file.endswith(".csv") is False: + raise Exception(f"File name is not valid (not a csv): {s3_file}") + s3_client = boto3.client('s3') - for s3_file in list_file_paths: - if s3_file.endswith(".csv") is False: - raise Exception(f"File name is not valid (not a csv): {s3_file}") - - # Manage the direction of the slashes just cases - s3_file = s3_file.replace("\\", "/") - if s3_file.startswith("/"): # remove it - s3_file = s3_file.lstrip("/") - - full_file_url = f"s3://{bucket_name}/{s3_file}" - if is_verbose: - print(f".. Downloading: {full_file_url}") + if s3_file.endswith(".csv") is False: + raise Exception(f"File name is not valid (not a csv): {s3_file}") - if rtn_df is None: - rtn_df = pd.read_csv(full_file_url) - continue + # Manage the direction of the slashes just cases + s3_file = s3_file.replace("\\", "/") - file_df = pd.read_csv(full_file_url) - rtn_df = pd.concat([r2f_df, file_df]) + if s3_file.startswith("/"): # remove it + s3_file = s3_file.lstrip("/") - print("") - print(f".. {len(list_file_paths)} files downloaded and loaded into the dataframe") + full_file_url = f"s3://{bucket_name}/{s3_file}" + if is_verbose: + print(f".. Downloading: {full_file_url}") + + rtn_df = pd.read_csv(full_file_url) + + print(f"... {len(list_file_paths)} files downloaded and loaded into the dataframe") return rtn_df From c8b88617f33faddd6626f98ad3bd6fee5699786c Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Tue, 3 Dec 2024 00:23:59 +0000 Subject: [PATCH 14/18] WIP added bridges --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 729 ++++++++++-------- .../helper_functions/s3_shared_functions.py | 23 +- 2 files changed, 397 insertions(+), 355 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index f446dc49..83590492 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -18,108 +18,40 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "9e1ee2b5-b109-49e8-8255-f06449b44ee6", "metadata": { "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: geopandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.0.1)\n", - "Requirement already satisfied: numpy>=1.22 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (1.26.4)\n", - "Requirement already satisfied: pyogrio>=0.7.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (0.10.0)\n", - "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (24.2)\n", - "Requirement already satisfied: pandas>=1.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.2.3)\n", - "Requirement already satisfied: pyproj>=3.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (3.7.0)\n", - "Requirement already satisfied: shapely>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (2.0.6)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2.9.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.2)\n", - "Requirement already satisfied: certifi in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pyogrio>=0.7.2->geopandas) (2024.8.30)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=1.4.0->geopandas) (1.16.0)\n", - "Requirement already satisfied: xarray in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (2024.10.0)\n", - "Requirement already satisfied: numpy>=1.24 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray) (1.26.4)\n", - "Requirement already satisfied: packaging>=23.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray) (24.2)\n", - "Requirement already satisfied: pandas>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray) (2.2.3)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray) (2.9.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray) (2024.2)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=2.1->xarray) (1.16.0)\n", - "Requirement already satisfied: geoalchemy2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (0.16.0)\n", - "Requirement already satisfied: SQLAlchemy>=1.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (2.0.36)\n", - "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (24.2)\n", - "Requirement already satisfied: typing-extensions>=4.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (4.12.2)\n", - "Requirement already satisfied: greenlet!=0.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (3.1.1)\n", - "Requirement already satisfied: rioxarray in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (0.18.1)\n", - "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (24.2)\n", - "Requirement already satisfied: rasterio>=1.3.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (1.4.2)\n", - "Requirement already satisfied: xarray>=2024.7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (2024.10.0)\n", - "Requirement already satisfied: pyproj>=3.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (3.7.0)\n", - "Requirement already satisfied: numpy>=1.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (1.26.4)\n", - "Requirement already satisfied: certifi in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pyproj>=3.3->rioxarray) (2024.8.30)\n", - "Requirement already satisfied: affine in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (2.4.0)\n", - "Requirement already satisfied: attrs in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (23.2.0)\n", - "Requirement already satisfied: click>=4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (8.1.7)\n", - "Requirement already satisfied: cligj>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (0.7.2)\n", - "Requirement already satisfied: click-plugins in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (1.1.1)\n", - "Requirement already satisfied: pyparsing in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (3.2.0)\n", - "Requirement already satisfied: pandas>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray>=2024.7.0->rioxarray) (2.2.3)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2.9.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2024.2)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=2.1->xarray>=2024.7.0->rioxarray) (1.16.0)\n", - "Requirement already satisfied: s3fs in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (0.4.2)\n", - "Requirement already satisfied: botocore>=1.12.91 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from s3fs) (1.35.63)\n", - "Requirement already satisfied: fsspec>=0.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from s3fs) (2024.10.0)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (1.0.1)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (2.9.0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (2.2.3)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil<3.0.0,>=2.1->botocore>=1.12.91->s3fs) (1.16.0)\n", - "Requirement already satisfied: python-dotenv in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.0.1)\n", - "*** All loaded ***\n" - ] - } - ], + "outputs": [], "source": [ "# Cell to manually pip reload a packages that the Jupyter engine not retained\n", - "# !pip install numpy\n", + "!pip install numpy\n", "!pip install geopandas\n", - "# !pip install pyarrow\n", "!pip install xarray\n", "!pip install geoalchemy2\n", - "# !pip install contextily\n", "!pip install rioxarray\n", "\n", + "# !pip install pyarrow\n", + "# !pip install contextily\n", + "\n", "!pip install s3fs # to get a newer version, old one on system\n", "!pip install python-dotenv\n", + "!pip install botocore\n", + "\n", "print(\"*** All loaded ***\")\n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 16, "id": "5b656259", "metadata": { "scrolled": true, "tags": [] }, "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, { "name": "stdout", "output_type": "stream", @@ -133,13 +65,14 @@ "import os\n", "import codecs\n", "import csv\n", - "\n", + "import datetime as dt\n", + "import shutil\n", "import sys\n", "\n", - "from datetime import datetime\n", "from pathlib import Path\n", "\n", "import boto3\n", + "import botocore\n", "import geopandas as gpd\n", "import json\n", "import pandas as pd\n", @@ -147,6 +80,7 @@ "import sqlalchemy\n", "import xarray as xr\n", "\n", + "\n", "from geopandas import GeoDataFrame\n", "from io import StringIO\n", "from geoalchemy2 import Geometry\n", @@ -162,8 +96,9 @@ "import helper_functions.s3_shared_functions as s3_sf\n", "\n", "from helper_functions.viz_classes import database\n", + "from helper_functions.viz_db_ingest import lambda_handler as execute_db_ingest\n", "\n", - "pd.set_option(\"max_info_rows\", 1000000) # override \n", + "pd.set_option(\"max_info_rows\", 1000000) # override\n", "print(\"option set\")\n", "\n", "print(\"imports loaded\")\n" @@ -171,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "fae5ca0f-e582-4ff2-89b6-087285433a97", "metadata": { "tags": [] @@ -208,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "e944eff3-6023-48f4-9f57-27304a240447", "metadata": { "tags": [] @@ -244,9 +179,8 @@ "# You might also see an error of 'An error occurred (NoSuchKey) when calling the GetObject operation:\n", "# The specified key does not exist.\" the creds are not correct\"\n", "\n", - "S3_CLIENT = boto3.client(\"s3\")\n", - "STEPFUNCTION_CLIENT = boto3.client('stepfunctions')\n", - "VIZ_DB_ENGINE = sf.get_db_engine('viz')\n", + "HYDROVIS_CRS = \"EPSG:3857\"\n", + "HYDROVIS_CRS_NUMBER = \"3857\"\n", "\n", "print(\"Global Variables loaded\")" ] @@ -278,11 +212,11 @@ "\n", "print(f\"Getting column name from {FIM_CROSSWALK_FPATH}\")\n", "\n", - "data = S3_CLIENT.get_object(Bucket=FIM_BUCKET, Key=FIM_CROSSWALK_FPATH)\n", + "s3_client = boto3.client(\"s3\")\n", + "data = s3_client.get_object(Bucket=FIM_BUCKET, Key=FIM_CROSSWALK_FPATH)\n", "d_reader = csv.DictReader(codecs.getreader(\"utf-8\")(data[\"Body\"]))\n", "headers = d_reader.fieldnames\n", "\n", - "\n", "header_str = \"(\"\n", "for header in headers:\n", " header_str += header\n", @@ -559,7 +493,8 @@ "# notice, slightly different object name\n", "pipeline_name = f\"hv_ti_data_loads_sage_aep_2_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", - "STEPFUNCTION_CLIENT.start_execution(\n", + "step_client = boto3.client('stepfunctions')\n", + "step_client.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", " name = pipeline_name,\n", " input= json.dumps(pipeline_input)\n", @@ -589,7 +524,8 @@ "# notice, slightly different object name\n", "pipeline_name = f\"hv_ti_data_loads_sage_aep_5_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", - "STEPFUNCTION_CLIENT.start_execution(\n", + "step_client = boto3.client('stepfunctions')\n", + "step_client.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", " name = pipeline_name,\n", " input= json.dumps(pipeline_input)\n", @@ -618,7 +554,8 @@ "# notice, slightly different object name\n", "pipeline_name = f\"hv_ti_data_loads_sage_aep_10_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", - "STEPFUNCTION_CLIENT.start_execution(\n", + "step_client = boto3.client('stepfunctions')\n", + "step_client.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", " name = pipeline_name,\n", " input= json.dumps(pipeline_input)\n", @@ -647,7 +584,8 @@ "# notice, slightly different object name\n", "pipeline_name = f\"hv_ti_data_loads_sage_aep_25_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", - "STEPFUNCTION_CLIENT.start_execution(\n", + "step_client = boto3.client('stepfunctions')\n", + "step_client.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", " name = pipeline_name,\n", " input= json.dumps(pipeline_input)\n", @@ -676,7 +614,8 @@ "# notice, slightly different object name\n", "pipeline_name = f\"hv_ti_data_loads_sage_aep_50_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", - "STEPFUNCTION_CLIENT.start_execution(\n", + "step_client = boto3.client('stepfunctions')\n", + "step_client.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", " name = pipeline_name,\n", " input= json.dumps(pipeline_input)\n", @@ -705,7 +644,8 @@ "# notice, slightly different object name\n", "pipeline_name = f\"hv_ti_data_loads_sage_aep_hw_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", - "STEPFUNCTION_CLIENT.start_execution(\n", + "step_client = boto3.client('stepfunctions')\n", + "step_client.start_execution(\n", " stateMachineArn = PIPELINE_ARN,\n", " name = pipeline_name,\n", " input= json.dumps(pipeline_input)\n", @@ -842,7 +782,7 @@ " ],\n", " \"sql_rename_dict\": {},\n", " \"logging_info\": {\n", - " \"Timestamp\": int(datetime.now().timestamp())\n", + " \"Timestamp\": int(dt.datetime.now().timestamp())\n", " }\n", " }\n", "\n", @@ -900,12 +840,13 @@ "outputs": [], "source": [ "\n", - "pipeline_name = f\"hv_ti_data_loads_catchments_branch_0_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_catchments_branch_0_{dt.datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "\n", "pipeline_input = get_catchment_pipepline_input(\"0\")\n", "# print(pipeline_input)\n", "\n", - "STEPFUNCTION_CLIENT.start_execution(\n", + "step_client = boto3.client('stepfunctions')\n", + "step_client.start_execution(\n", " stateMachineArn=PIPELINE_ARN,\n", " name=pipeline_name,\n", " input=json.dumps(pipeline_input)\n", @@ -959,11 +900,12 @@ "outputs": [], "source": [ "\n", - "pipeline_name = f\"hv_ti_data_loads_catchments_gms_{datetime.now().strftime('%Y%m%dT%H%M')}\"\n", + "pipeline_name = f\"hv_ti_data_loads_catchments_gms_{dt.datetime.now().strftime('%Y%m%dT%H%M')}\"\n", "pipeline_input = get_catchment_pipepline_input(\"gms\")\n", "# print(pipeline_input)\n", "\n", - "STEPFUNCTION_CLIENT.start_execution(\n", + "step_client = boto3.client('stepfunctions')\n", + "step_client.start_execution(\n", " stateMachineArn=PIPELINE_ARN,\n", " name=pipeline_name,\n", " input=json.dumps(pipeline_input)\n", @@ -976,7 +918,7 @@ }, { "cell_type": "markdown", - "id": "05fb7850-5be1-4127-86ba-bbfde273424a", + "id": "ec81c974-8a3c-416b-9daa-2aaf7d577a25", "metadata": { "tags": [] }, @@ -984,14 +926,75 @@ "

7 - Recreate derived.usgs_elev_table

" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "c855b07d-ef8b-4dc6-94a7-e6d0f2faf279", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb366d43-9aed-4487-841c-9888d701f6fb", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a22955a9-eb15-4065-9825-ccb964cc9c66", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aeb75842-ede9-4ea8-8fac-43b21b27de14", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d6a7842-2a24-4c5b-9fab-20ae3355c387", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff2b74e2-bde9-43bc-8548-e6277e439ecc", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee4bdcd7-ce35-44e4-affa-64f827e47399", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c85a0d4a-4af1-495b-b837-780a485c8a95", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", "metadata": { - "jupyter": { - "source_hidden": true - }, "scrolled": true, "tags": [] }, @@ -1004,7 +1007,9 @@ "\n", "uet_usecols = ['location_id', 'HydroID', 'dem_adj_elevation', 'nws_lid', 'levpa_id']\n", "\n", - "paginator = S3_CLIENT.get_paginator('list_objects')\n", + "viz_db_engine = sf.get_db_engine('viz')\n", + "s3_client = boto3.client(\"s3\")\n", + "paginator = s3_client.get_paginator('list_objects')\n", "operation_parameters = {'Bucket': FIM_BUCKET,\n", " 'Prefix': f'{HAND_DATASETS_DPATH}/',\n", " 'Delimiter': '/'}\n", @@ -1018,11 +1023,11 @@ " for i, prefix_obj in enumerate(prefix_objects):\n", " rec_num = (i + 1) + (1000 * page_count)\n", " print(f\"Processing rec number {rec_num} : \"\n", - " f\" On page {page_count} of {num_pages} (up to 1000 per page)\")\n", + " f\" On page {page_count + 1} of appx 2,150 recs (up to 1000 per page)\")\n", " huc_prefix = prefix_obj.get(\"Prefix\")\n", " usgs_elev_table_key = f'{huc_prefix}usgs_elev_table.csv'\n", " try:\n", - " uet = S3_CLIENT.get_object(\n", + " uet = s3_client.get_object(\n", " Bucket=FIM_BUCKET, \n", " Key=usgs_elev_table_key\n", " )['Body']\n", @@ -1030,7 +1035,7 @@ " # uet_df['fim_version'] = PUBLIC_FIM_VERSION\n", " uet_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", " uet_df.to_sql(\n", - " con=VIZ_DB_ENGINE,\n", + " con=viz_db_engine,\n", " dtype={\n", " \"location_id\": Text(),\n", " \"nws_data_huc\": Text()\n", @@ -1085,7 +1090,9 @@ "# df = sf.sql_to_dataframe(sql)\n", "# ht_usecols = ['HydroID', 'feature_id', 'stage', 'discharge_cms']\n", "\n", - "# paginator = S3_CLIENT.get_paginator('list_objects')\n", + "# viz_db_engine = sf.get_db_engine('viz')\n", + "# s3_client = boto3.client(\"s3\")\n", + "# paginator = s3_client.get_paginator('list_objects')\n", "# operation_parameters = {'Bucket': FIM_BUCKET,\n", "# 'Prefix': f'{HAND_DATASETS_DPATH}/',\n", "# 'Delimiter': '/'}\n", @@ -1131,7 +1138,7 @@ "# f\" On page {page_count + 1} of (number of pages unknown) - (up to 1000 per page)\")\n", "\n", "# branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", - "# branch_files_result = S3_CLIENT.list_objects(\n", + "# branch_files_result = s3_client.list_objects(\n", "# Bucket=FIM_BUCKET, \n", "# Prefix=branch_prefix, \n", "# Delimiter='/'\n", @@ -1147,7 +1154,7 @@ "# # print(f\"Found usgs_elev_table and hydroTable in {branch_prefix}\")\n", "# try:\n", "# # print(\"...Fetching csvs...\")\n", - "# ht = S3_CLIENT.get_object(\n", + "# ht = s3_client.get_object(\n", "# Bucket=FIM_BUCKET,\n", "# Key=hydro_table_key\n", "# )['Body']\n", @@ -1186,7 +1193,7 @@ "# print(\".... saving dataframe to db\")\n", "# df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", "# df.to_sql(\n", - "# con=VIZ_DB_ENGINE, \n", + "# con=viz_db_engine, \n", "# schema='derived',\n", "# name='hydrotable',\n", "# index=False,\n", @@ -1224,190 +1231,23 @@ "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hydrotable reloaded - started\n", - "Processing rec number 1 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 2 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 3 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 4 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 5 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 6 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 7 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 8 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 9 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 10 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 11 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 12 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 13 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 14 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 15 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 16 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 17 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 18 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 19 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 20 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 21 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 22 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 23 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 24 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 25 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 26 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 27 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 28 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 29 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 30 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 31 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 32 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 33 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 34 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 35 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 36 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 37 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 38 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 39 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 40 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 41 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 42 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 43 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 44 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 45 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 46 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 47 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 48 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 49 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 50 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 51 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 52 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 53 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 54 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 55 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 56 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 57 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 58 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 59 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 60 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 61 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 62 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 63 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 64 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 65 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 66 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 67 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 68 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 69 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 70 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 71 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 72 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 73 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 74 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 75 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 76 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 77 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 78 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 79 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 80 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 81 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 82 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 83 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 84 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 85 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 86 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 87 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 88 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 89 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 90 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 91 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 92 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 93 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 94 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 95 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 96 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 97 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 98 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 99 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 100 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 101 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 102 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 103 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 104 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 105 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 106 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 107 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 108 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 109 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 110 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 111 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 112 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 113 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 114 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 115 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 116 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 117 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 118 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 119 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 120 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 121 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 122 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 123 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 124 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 125 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 126 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 127 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 128 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 129 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 130 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 131 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 132 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 133 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 134 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 135 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 136 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 137 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 138 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 139 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 140 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 141 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 142 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 143 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 144 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 145 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 146 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 147 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 148 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 149 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 150 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 151 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 152 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 153 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 154 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 155 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 156 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 157 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 158 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 159 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 160 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 161 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 162 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 163 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 164 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 165 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 166 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 167 : On page 1 (appx 2150 recs in total)\n", - "Processing rec number 168 : On page 1 (appx 2150 recs in total)\n" - ] - } - ], + "outputs": [], "source": [ "\n", - "# Note: This version finished in 1hr 49m but had to be babysat, clicking on the screen very 10 - 15 mins\n", + "# Note version ws even slower, it took just under 8 hrs to run (output messages?)\n", + "\n", + "\n", + "# ++++++++++++++\n", + "# FOR THE NEXT VERSION.. Don't load this against each branch. The branches are all\n", + "# aggregated into a HUC level hydrotable.\n", + "# ++++++++++++++\n", + "\n", "\n", "print(\"hydrotable reloaded - started\")\n", - "start_dt = datetime.now()\n", + "start_dt = dt.datetime.now()\n", + "print(f\"Started: {dt.datetime.now().strftime('%m/%d/%Y, %H:%M:%S')}\")\n", "\n", + "viz_db_engine = sf.get_db_engine('viz')\n", "sf.execute_sql('DROP TABLE IF EXISTS derived.hydrotable;')\n", "sql = '''\n", "SELECT distinct LPAD(huc8::text, 8, '0') as huc8 FROM derived.featureid_huc_crosswalk WHERE huc8 is not null;\n", @@ -1415,7 +1255,8 @@ "df = sf.sql_to_dataframe(sql)\n", "ht_usecols = ['HydroID', 'feature_id', 'stage', 'discharge_cms']\n", "\n", - "paginator = S3_CLIENT.get_paginator('list_objects')\n", + "s3_client = boto3.client(\"s3\")\n", + "paginator = s3_client.get_paginator('list_objects')\n", "operation_parameters = {'Bucket': FIM_BUCKET,\n", " 'Prefix': f'{HAND_DATASETS_DPATH}/',\n", " 'Delimiter': '/'}\n", @@ -1436,11 +1277,12 @@ " total_recs_in_page = len(prefix_objects)\n", "\n", " rec_num = (i + 1) + (1000 * page_count)\n", + " display_dt = dt.datetime.now().strftime(\"%m/%d/%Y, %H:%M:%S\")\n", " print(f\"Processing rec number {rec_num} : \"\n", - " f\" On page {page_count + 1} (appx 2150 recs in total)\")\n", + " f\" On page {page_count + 1} (appx 2150 recs in total) : {display_dt}\")\n", "\n", " branch_prefix = f'{prefix_obj.get(\"Prefix\")}branches/0/'\n", - " branch_files_result = S3_CLIENT.list_objects(\n", + " branch_files_result = s3_client.list_objects(\n", " Bucket=FIM_BUCKET, \n", " Prefix=branch_prefix, \n", " Delimiter='/'\n", @@ -1450,29 +1292,32 @@ " branch_file_prefix = content_obj['Key']\n", " if 'hydroTable' in branch_file_prefix:\n", " hydro_table_key = branch_file_prefix\n", + " break\n", "\n", " if hydro_table_key:\n", "\n", " # print(f\"Found usgs_elev_table and hydroTable in {branch_prefix}\")\n", " try:\n", - " # print(\"...Fetching csvs...\")\n", - " ht = S3_CLIENT.get_object(\n", + " print(f\"...Fetching and loading csv for {hydro_table_key} ...\")\n", + " ht = s3_client.get_object(\n", " Bucket=FIM_BUCKET,\n", " Key=hydro_table_key\n", " )['Body']\n", " # print(\"... Reading with pandas ...\")\n", " ht_df = pd.read_csv(ht, header=0, usecols=ht_usecols)\n", + " ht = None\n", "\n", " # print(\"... saving dataframe to db\")\n", " ht_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", " ht_df.to_sql(\n", - " con=VIZ_DB_ENGINE,\n", + " con=viz_db_engine,\n", " schema='derived',\n", " name='hydrotable',\n", " index=False,\n", " if_exists='append',\n", " method='multi'\n", " )\n", + " ht_df = None\n", " # print(\"... saved to db\")\n", "\n", " except Exception as e:\n", @@ -1481,10 +1326,11 @@ "\n", " page_count += 1\n", "\n", - "end_dt = datetime.now()\n", + "end_dt = dt.datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(\"***************\")\n", "print(\"hydrotable reload done\")\n", + "print(f\"Ended: {dt.datetime.now().strftime('%m/%d/%Y, %H:%M:%S')}\")\n", "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", "print(\"\")\n" ] @@ -1498,10 +1344,12 @@ }, "outputs": [], "source": [ + "\n", + "# takes only a few mins\n", "\n", "print(\"hydrotable_staggered started\")\n", "\n", - "start_dt = datetime.now()\n", + "start_dt = dt.datetime.now()\n", "\n", "sql = '''\n", "DROP TABLE IF EXISTS derived.hydrotable_staggered;\n", @@ -1519,7 +1367,7 @@ "sf.execute_sql(sql)\n", "\n", "print(\"hydrotable_staggered reload done\")\n", - "end_dt = datetime.now()\n", + "end_dt = dt.datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", "\n" @@ -1530,6 +1378,9 @@ "execution_count": null, "id": "74f21c03-ca36-402c-a69b-396162720f66", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1555,6 +1406,9 @@ "execution_count": null, "id": "461f20b4-8442-4c0e-9114-ea62764cd789", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1575,6 +1429,9 @@ "execution_count": null, "id": "b752fce5-c8d7-4064-92d7-842c22d1723e", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1594,27 +1451,34 @@ "execution_count": null, "id": "d6a0cc18-de82-4bd8-bd4d-c3ad021a1dec", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], "source": [ "# run the script to load the usgs_rating_curve.csv. Exact duration not yet known. Appx 30 min (??)\n", "\n", - "start_dt = datetime.now()\n", + "print(\"Starting loading usgs_rating_curves. Takes appx 30 mins\")\n", + "\n", + "start_dt = dt.datetime.now()\n", + "ref_time = dt.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n", + "print(f\"ref_time is {ref_time}\")\n", "event = {\n", " 'target_table': 'derived.usgs_rating_curves',\n", " 'target_cols': ['location_id', 'flow', 'stage', 'navd88_datum', 'elevation_navd88'],\n", " 'file': f'{QA_DATASETS_DPATH}/usgs_rating_curves.csv',\n", " 'bucket': FIM_BUCKET,\n", - " 'reference_time': '2023-08-23 00:00:00',\n", + " 'reference_time': f'{ref_time}',\n", " 'keep_flows_at_or_above': 0,\n", " 'iteration_index': 0\n", "}\n", "\n", - "sf.execute_db_ingest(event, None)\n", + "execute_db_ingest(event, None)\n", "\n", "print(\"done loading usgs_rating_curves\")\n", - "end_dt = datetime.now()\n", + "end_dt = dt.datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" ] @@ -1624,6 +1488,9 @@ "execution_count": null, "id": "4548d5e8-303e-44e0-a8a5-7b1947214b4e", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1655,6 +1522,9 @@ "execution_count": null, "id": "d1a77875-f569-4410-888c-b132714a7c9e", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1680,13 +1550,16 @@ "execution_count": null, "id": "c24f509a-90df-492a-a0d4-aa9eab8e7041", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], "source": [ "\n", "# Load the src_skill_temp table\n", - "start_dt = datetime.now()\n", + "start_dt = dt.datetime.now()\n", "\n", "sf.eecute_sql('DROP TABLE IF EXISTS derived.src_skill_temp;', db_type='viz')\n", "\n", @@ -1710,7 +1583,7 @@ ")\n", "\n", "print(\"Done loading derived.src_skill_temp table\")\n", - "end_dt = datetime.now()\n", + "end_dt = dt.datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" ] @@ -1728,7 +1601,7 @@ "# Load into src_skill table adding geometry to it from external.usgs_gage. Yes.. more/less straight from WRDS tables\n", "# Some recs appear to be in error in the csv. location id = 394220106431500 (those are dropped below)\n", "\n", - "start_dt = datetime.now()\n", + "start_dt = dt.datetime.now()\n", "\n", "sf.execute_sql('DROP TABLE IF EXISTS derived.src_skill;', db_type='viz')\n", "\n", @@ -1751,7 +1624,7 @@ "sf.execute_sql(sql)\n", "\n", "print(\"Done loading derived.src_skill table\")\n", - "end_dt = datetime.now()\n", + "end_dt = dt.datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", "\n" @@ -1772,6 +1645,9 @@ "execution_count": null, "id": "0df23345-47a4-490a-b10f-2020e3a2a68e", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -1833,13 +1709,13 @@ "\n", "# Load the new fim performance tables \n", "\n", - "start_dt = datetime.now() # appx 35 min\n", + "start_dt = dt.datetime.now() # appx 35 min\n", "\n", "# os.environ['EGIS_DB_HOST'] ='' #TI DB\n", "\n", "db_type = \"egis\"\n", "db_engine = sf.get_db_engine(db_type)\n", - "s3 = boto3.client('s3')\n", + "# s3 = boto3.client('s3')\n", "\n", "# Define bucket and parent directories.\n", "bucket = \"hydrovis-ti-deployment-us-east-1\"\n", @@ -1867,7 +1743,7 @@ " if file_handle == 'fim_performance_points.csv':\n", " df = df.rename(columns={'Unnamed: 0': 'oid', 'geometry': 'geom'})\n", " else:\n", - " df = df.rename(columns={'Unnamed: 0': 'oid', 'geometry': 'geom', 'huc':'huc8'})\n", + " df = df.rename(columns={'Unnamed: 0': 'oid', 'geometry': 'geom', 'huc': 'huc8'})\n", "\n", " # print(df.dtypes)\n", " # Convert all field names to lowercase (needed for ArcGIS Pro).\n", @@ -1934,7 +1810,7 @@ " index=False,\n", " dtype={'oid': sqlalchemy.types.Integer(),\n", " 'version': sqlalchemy.types.String(),\n", - " 'geom': Geometry('MULTIPOLYGON', srid=3857)\n", + " 'geom': Geometry('MULTIPOLYGON', srid=HYDROVIS_CRS_NUMBER)\n", " }\n", " )\n", " # Load remaining chunks into newly created table\n", @@ -1949,7 +1825,7 @@ " index=False,\n", " dtype={'oid': sqlalchemy.types.Integer(),\n", " 'version': sqlalchemy.types.String(),\n", - " 'geom': Geometry('MULTIPOLYGON', srid=3857)\n", + " 'geom': Geometry('MULTIPOLYGON', srid=HYDROVIS_CRS_NUMBER)\n", " }\n", " )\n", " else:\n", @@ -1965,7 +1841,7 @@ " index=False,\n", " dtype={'oid': sqlalchemy.types.Integer(),\n", " 'version': sqlalchemy.types.String(),\n", - " 'geom': Geometry(geometry, srid=3857)\n", + " 'geom': Geometry(geometry, srid=HYDROVIS_CRS_NUMBER)\n", " }\n", " )\n", "\n", @@ -1978,7 +1854,7 @@ "\n", "print(\"\")\n", "\n", - "end_dt = datetime.now()\n", + "end_dt = dt.datetime.now()\n", "time_duration = end_dt - start_dt\n", "# print(\"All FIM Performance files loaded\")\n", "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" @@ -2018,15 +1894,12 @@ "\n", " '''\n", " Inputs:\n", - " - catfim_type: name identififer for the set, such as \"flow_based_catfim\" or \"flow_based_catfim_sites\", etc\n", + " - catfim_type: name identififer for the set, such as \"flow_based_catfim_library\" or \"flow_based_catfim_sites\", etc\n", " Sometimes the file_handle name can be the name of the s3 file (without extension) and/or the table\n", " name.\n", - " Options: flow_based_catfim, flow_based_catfim_sites, stage_based_catfim, stage_based_catfim_sites\n", " '''\n", "\n", " db_type = \"egis\"\n", - " db_engine = sf.get_db_engine(db_type)\n", - " src_crs = \"3857\"\n", "\n", " # --------------------------------------\n", " # Drop the original Db if already in place\n", @@ -2038,26 +1911,32 @@ "\n", " # --------------------------------------\n", " # Get the data from S3 and load it into a df\n", - " if catfim_type in ['flow_based_catfim', 'stage_based_catfim']:\n", - " file_to_download = f\"{QA_DATASETS_DPATH}/{catfim_type}_library.csv\"\n", - " else:\n", - " file_to_download = f\"{QA_DATASETS_DPATH}/{catfim_type}.csv\"\n", + " file_to_download = f\"{QA_DATASETS_DPATH}/{catfim_type}.csv\"\n", "\n", " # print(f\"Downloading {file_to_download} ... \")\n", "\n", - " df = s3_sf.download_S3_csv_files_to_df_from_list(FIM_BUCKET, [file_to_download], True)\n", + " df = s3_sf.load_S3_csv_to_df(FIM_BUCKET, file_to_download, True)\n", " num_recs = len(df)\n", " print(f\"File read. {num_recs} records to load\")\n", "\n", " # --------------------------------------\n", " # Adjusting Columns and data\n", " # Rename headers. All files this name\n", - " df = df.rename(columns={'Unnamed: 0': 'oid',\n", - " 'geometry': 'geom',\n", - " 'huc': 'huc8'})\n", "\n", " # Convert all field names to lowercase (needed for ArcGIS Pro).\n", " df.columns = df.columns.str.lower()\n", + " \n", + " df = df.rename(columns={'geometry': 'geom'})\n", + "\n", + " if \"fid\" in df.columns:\n", + " df = df.rename(columns={'fid': 'oid'})\n", + " else:\n", + " df = df.rename(columns={'Unnamed: 0': 'oid',\n", + " 'geometry': 'geom'})\n", + "\n", + " if \"huc\" in df.columns:\n", + " df = df.rename(columns={'huc': 'huc8'})\n", + "\n", "\n", " # Enforce data types on df before loading in DB (TODO: need to create special cases for each layer).\n", " df = df.astype({'huc8': 'str'})\n", @@ -2086,16 +1965,20 @@ " df = df.astype({'nws_data_rfc_defined_fcst_point': 'str'})\n", " df = df.astype({'nws_data_riverpoint': 'str'})\n", "\n", + " if \"version\" in df.columns:\n", + " df.drop(\"version\", axis=1, inplace=True)\n", + " if \"viz\" in df.columns:\n", + " df.drop(\"viz\", axis=1, inplace=True)\n", "\n", - " # As of Nov 1, 2024: Ignore the incoming \"version\" from dataset\n", - " # df['version'] = PUBLIC_FIM_VERSION\n", - " df[COLUMN_NAME_FIM_VERSION] = PUBLIC_FIM_VERSION\n", - " df[COLUMN_NAME_MODEL_VERSION] = FIM_MODEL_VERSION\n", + " df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", "\n", " # --------------------------------------\n", " # Load to DB\n", " # Chunk load data into DB\n", - " if catfim_type in ['flow_based_catfim', 'stage_based_catfim']:\n", + "\n", + " db_engine = sf.get_db_engine(db_type)\n", + "\n", + " if catfim_type in ['flow_based_catfim_library', 'stage_based_catfim_library']: # Libraries\n", "\n", " # Create list of df chunks\n", " n = 1000 # chunk row size\n", @@ -2173,6 +2056,9 @@ "execution_count": null, "id": "f9577edb-4aa6-423b-819e-df8c922c7ec2", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2248,19 +2134,25 @@ "\n", "print(\"Starting of CatFIM data\")\n", "\n", - "# catfim_types = ['flow_based_catfim', 'flow_based_catfim_sites']\n", - "# catfim_types = ['stage_based_catfim', 'stage_based_catfim_sites']\n", - "catfim_types = ['stage_based_catfim_sites']\n", - "# catfim_types = ['stage_based_catfim']\n", + "# the two sites csv's take a few seconds to laod\n", + "# Stagae based library takes appx 6 - 7 Mins\n", + "# flow based library takes apppx\n", + "\n", + "# catfim_types = ['flow_based_catfim_sites']\n", + "catfim_types = ['flow_based_catfim_library']\n", + "# catfim_types = ['flow_based_catfim_library', 'flow_based_catfim_sites']\n", + "# catfim_types = ['stage_based_catfim'_library, 'stage_based_catfim_sites']\n", + "# catfim_types = ['stage_based_catfim_sites']\n", + "# catfim_types = ['stage_based_catfim_library']\n", "\n", - "start_dt = datetime.now()\n", + "start_dt = dt.datetime.now()\n", "\n", "for catfim_type in catfim_types:\n", " print(f\"Loading {catfim_type} data\")\n", " load_catfim_table(catfim_type)\n", "\n", "print(\"\")\n", - "end_dt = datetime.now()\n", + "end_dt = dt.datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n", "\n" @@ -2281,6 +2173,9 @@ "execution_count": null, "id": "f5cf108d-7360-48a9-a4a2-81b23b9e51b4", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -2346,7 +2241,7 @@ "\n", "__public_fim_release = \"fim_30\" # The new fim public release being loaded (ie. fim_10, fim_30, fim_60..)\n", "\n", - "start_dt = datetime.now()\n", + "start_dt = dt.datetime.now()\n", "\n", "for catfim_type in catfim_types:\n", " print(\"\")\n", @@ -2372,19 +2267,169 @@ "# print(f\"{db_name} copied to {new_table_name} if it does not already exist\")\n", "\n", "print(\"\")\n", - "end_dt = datetime.now()\n", + "end_dt = dt.datetime.now()\n", + "time_duration = end_dt - start_dt\n", + "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "5ee92624-92bd-4c6d-ad24-c1d0b840491d", + "metadata": {}, + "source": [ + "

13 - Load Bridge Centroids Data

" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "59c382de-d616-40f4-8aa5-e7cdcc8010de", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Bridge Centroids data load started\n", + "============================\n", + "Removing all temp files\n", + "\n", + "Bridge Centroids data load completed\n", + "... duration was 0:00:00\n" + ] + } + ], + "source": [ + "\n", + "# Has appx 2,150 HUCs to process, but this section goes quickly.\n", + "\n", + "print(\"Bridge Centroids data load started\")\n", + "start_dt = dt.datetime.now()\n", + "\n", + "sf.execute_sql('DROP TABLE IF EXISTS derived.bridge_centroidS;')\n", + "\n", + "# Set temp folder and file path for each gpkg to download too in order to laod\n", + "# In later versions, we will have FIM-dev to create them as csv's but for now\n", + "# we have to download the gpkg to a local temp drive space and file, the reload it\n", + "# We will just keep writing over the same file, no need to keep any temp file\n", + "trg_path = os.path.join(Path.home(), \"SageMaker\", \"temp_files\", \"bridges\")\n", + "\n", + "# folders may not yet exist\n", + "if not os.path.exists(trg_path):\n", + " os.makedirs(trg_path, exist_ok=True)\n", + "\n", + "s3_session = boto3.session.Session(aws_access_key_id=TI_ACCESS_KEY,\n", + " aws_secret_access_key=TI_SECRET_KEY,\n", + " aws_session_token=TI_TOKEN,\n", + " region_name='us-east-1')\n", + "\n", + "s3_client = s3_session.client('s3')\n", + "viz_db_engine = sf.get_db_engine('viz')\n", + "\n", + "paginator = s3_client.get_paginator('list_objects_v2')\n", + "operation_parameters = {'Bucket': FIM_BUCKET,\n", + " 'Prefix': f'{HAND_DATASETS_DPATH}/',\n", + " 'Delimiter': '/'}\n", + "page_iterator = paginator.paginate(**operation_parameters)\n", + "page_count = 0\n", + "debug_break = False\n", + "\n", + "print(f\"... loading from s3://{FIM_BUCKET}/{HAND_DATASETS_DPATH}\")\n", + "print(\"\")\n", + "\n", + "for page in page_iterator:\n", + "\n", + " # Temp debugging early breaking\n", + " # if debug_break:\n", + " # break\n", + "\n", + " prefix_objects = page['CommonPrefixes']\n", + " for i, prefix_obj in enumerate(prefix_objects):\n", + "\n", + " try:\n", + " print(\"-------------------------------------------\")\n", + "\n", + " huc_prefix = prefix_obj.get(\"Prefix\")\n", + " bridges_file_key = f'{huc_prefix}osm_bridge_centroids.gpkg'\n", + "\n", + " # print(f\"bridges_file_key is {bridges_file_key}\")\n", + " # ensure the file exists (yes.. some people don't like this convention but it is pretty fast and safe)\n", + " try:\n", + " s3_client.head_object(Bucket=FIM_BUCKET, Key=bridges_file_key)\n", + "\n", + " except botocore.exceptions.ClientError as e:\n", + " if e.response['Error']['Code'] == \"404\":\n", + " print(f\"Rec number {(i + 1)} not found and skipped (and it is fine)\")\n", + " continue\n", + " else:\n", + " # Something else has gone wrong.\n", + " raise e\n", + "\n", + " rec_num = (i + 1) + (1000 * page_count)\n", + " print(f\"Processing rec number {rec_num} of appx 2,150\")\n", + "\n", + " huc8 = huc_prefix[:-1].split('/')[-1]\n", + " huc2 = huc8[:2]\n", + " temp_gpkg_file = os.path.join(trg_path, f\"{huc8}.gpkg\")\n", + "\n", + " print(f\"... Loading /{bridges_file_key}\")\n", + " print(f\"........ to {temp_gpkg_file}\")\n", + "\n", + " s3_client.download_file(FIM_BUCKET, bridges_file_key, temp_gpkg_file)\n", + "\n", + " # # now we can open it from the file system\n", + " bridges_df = gpd.read_file(temp_gpkg_file)\n", + "\n", + " # All CONUS+ hucs come from FIM as EPSG:5070, except AK which is EPSG:3338.\n", + " # we need to reproject all to 3857\n", + " reproj_bridges_df = bridges_df.to_crs(HYDROVIS_CRS)\n", + "\n", + " reproj_bridges_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", + " reproj_bridges_df.to_postgis(\n", + " con=viz_db_engine,\n", + " schema='derived',\n", + " name='bridge_centroids',\n", + " index=True,\n", + " index_label='oid',\n", + " if_exists='append',\n", + " dtype={'geom': Geometry('POINT', srid=HYDROVIS_CRS_NUMBER)}\n", + " )\n", + "\n", + " # We will break after the first record to debug\n", + " # if i >= 0:\n", + " # debug_break = True\n", + " # break\n", + "\n", + " except Exception as e:\n", + " if \"NoSuchKey\" in str(e):\n", + " pass\n", + " else:\n", + " raise e\n", + "\n", + "# page_count += 1\n", + "\n", + "######\n", + "# remove all of the temp bridge files \n", + "print(\"============================\")\n", + "if os.path.exists(trg_path):\n", + " print(\"Removing all temp files\")\n", + " shutil.rmtree(trg_path)\n", + "\n", + "print(\"\")\n", + "print(\"Bridge Centroids data load completed\")\n", + "end_dt = dt.datetime.now()\n", "time_duration = end_dt - start_dt\n", "print(f\"... duration was {str(time_duration).split('.')[0]}\")\n" ] }, { "cell_type": "markdown", - "id": "eb36b537-f957-457a-9bfa-64d944f85599", + "id": "7acbbe20-cb66-44f1-a3c6-9df366e25008", "metadata": { "tags": [] }, "source": [ - "

13 - Clear the HAND Cache

" + "

14 - Clear the HAND Cache

" ] }, { @@ -2407,12 +2452,12 @@ }, { "cell_type": "markdown", - "id": "166702e8-60d2-485e-a5bc-23b475841e5a", + "id": "bc1e1bc3-de1e-45b9-9036-e0abe90a0e92", "metadata": { "tags": [] }, "source": [ - "

14 - SAVE TO REPO (AND REDEPLOY TO TI WITH NEW VERSION VARIABLE IN TERRAFORM ??)

\n", + "

15 - SAVE TO REPO (AND REDEPLOY TO TI WITH NEW VERSION VARIABLE IN TERRAFORM ??)

\n", "\n", "Oct 21, 2024: We don''t have a system per-say to update for Terraform, but we now have github hooks\n", "built right into JupyterHub. We need to figure out how to work with multiple branches and \"getting latest\"\n", diff --git a/Core/Manual_Workflows/helper_functions/s3_shared_functions.py b/Core/Manual_Workflows/helper_functions/s3_shared_functions.py index 9cd99d06..141ee015 100644 --- a/Core/Manual_Workflows/helper_functions/s3_shared_functions.py +++ b/Core/Manual_Workflows/helper_functions/s3_shared_functions.py @@ -100,11 +100,11 @@ # ***************************** # CAUTION: TODO: Aug 2024: This needs to be re-thought. I can easily overpower the notebook server depending on teh size of the notebooks # ***************************** -def load_S3_csv_to_df(s3_file_path, is_verbose=False): +def load_S3_csv_to_df(bucket_name, s3_file_path, is_verbose=False): ''' Overview: - - a full s3 file path - ie) s3://hydrovis-ti-deployment-us-east-1/fim/hand_4_5_11_1/qa_datasets/fim_performance_catchments.csv + - bucket: ie hydrovis-ti-deployment-us-east-1 + - s3+file_path: ie) fim/hand_4_5_11_1/qa_datasets/fim_performance_catchments.csv Returns: - A dataframe with the csv data loaded ''' @@ -112,28 +112,25 @@ def load_S3_csv_to_df(s3_file_path, is_verbose=False): if s3_file_path is None or s3_file_path == '' or len(s3_file_path) < 5: raise Exception("s3_file_path is invalid - not set, empty or too short") - if s3_file.endswith(".csv") is False: + if s3_file_path.endswith(".csv") is False: raise Exception(f"File name is not valid (not a csv): {s3_file}") s3_client = boto3.client('s3') - if s3_file.endswith(".csv") is False: - raise Exception(f"File name is not valid (not a csv): {s3_file}") - # Manage the direction of the slashes just cases - s3_file = s3_file.replace("\\", "/") + s3_file_path = s3_file_path.replace("\\", "/") - if s3_file.startswith("/"): # remove it - s3_file = s3_file.lstrip("/") + if s3_file_path.startswith("/"): # remove it + s3_file_path = s3_file.lstrip("/") - full_file_url = f"s3://{bucket_name}/{s3_file}" + full_file_url = f"s3://{bucket_name}/{s3_file_path}" if is_verbose: print(f".. Downloading: {full_file_url}") rtn_df = pd.read_csv(full_file_url) - print(f"... {len(list_file_paths)} files downloaded and loaded into the dataframe") - + print(".. Downloading complete") + return rtn_df From d2443c6ab465db5b09dcbb58ea532e5a23807781 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Tue, 10 Dec 2024 22:54:12 +0000 Subject: [PATCH 15/18] cleanup outputs plus run cache trunc --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 8947 +---------------- 1 file changed, 24 insertions(+), 8923 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 32bb424b..3390df7f 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -2,10 +2,10 @@ "cells": [ { "cell_type": "markdown", - "id": "68fce701-8ba2-400d-b662-dca6439ef9b0", + "id": "2291579e-30af-4da8-8cb5-28a6f76a2635", "metadata": {}, "source": [ - "### Notes - Oct 21, 2024 ###\n", + "### Notes - Oct 21, 2024 to Dec 10, 2024 ###\n", "This is a copy from 10.FIM Version 4.5.2.11 which included hand data loads plus ras2fim data. \n", "We will remove all ras2fim stuff here knowing that sometimes ras2fim will be uploaded on its own.\n", "However.. when ras2fim is loaded, some steps here will need to be re-run. Those steps will be\n", @@ -18,137 +18,13 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "9e1ee2b5-b109-49e8-8255-f06449b44ee6", "metadata": { "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: numpy in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.26.4)\n", - "Collecting geopandas\n", - " Downloading geopandas-1.0.1-py3-none-any.whl.metadata (2.2 kB)\n", - "Requirement already satisfied: numpy>=1.22 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (1.26.4)\n", - "Collecting pyogrio>=0.7.2 (from geopandas)\n", - " Downloading pyogrio-0.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.5 kB)\n", - "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (21.3)\n", - "Requirement already satisfied: pandas>=1.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geopandas) (1.5.3)\n", - "Collecting pyproj>=3.3.0 (from geopandas)\n", - " Downloading pyproj-3.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (31 kB)\n", - "Collecting shapely>=2.0.0 (from geopandas)\n", - " Downloading shapely-2.0.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.0 kB)\n", - "Requirement already satisfied: python-dateutil>=2.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2.9.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.2)\n", - "Requirement already satisfied: certifi in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pyogrio>=0.7.2->geopandas) (2024.8.30)\n", - "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from packaging->geopandas) (3.2.0)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas>=1.4.0->geopandas) (1.16.0)\n", - "Downloading geopandas-1.0.1-py3-none-any.whl (323 kB)\n", - "Downloading pyogrio-0.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (23.7 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m137.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hDownloading pyproj-3.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.2 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.2/9.2 MB\u001b[0m \u001b[31m111.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hDownloading shapely-2.0.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.5 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.5/2.5 MB\u001b[0m \u001b[31m126.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hInstalling collected packages: shapely, pyproj, pyogrio, geopandas\n", - "Successfully installed geopandas-1.0.1 pyogrio-0.10.0 pyproj-3.7.0 shapely-2.0.6\n", - "Collecting xarray\n", - " Downloading xarray-2024.11.0-py3-none-any.whl.metadata (11 kB)\n", - "Requirement already satisfied: numpy>=1.24 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray) (1.26.4)\n", - "Collecting packaging>=23.2 (from xarray)\n", - " Downloading packaging-24.2-py3-none-any.whl.metadata (3.2 kB)\n", - "Collecting pandas>=2.1 (from xarray)\n", - " Downloading pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (89 kB)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray) (2.9.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray) (2024.2)\n", - "Collecting tzdata>=2022.7 (from pandas>=2.1->xarray)\n", - " Using cached tzdata-2024.2-py2.py3-none-any.whl.metadata (1.4 kB)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=2.1->xarray) (1.16.0)\n", - "Downloading xarray-2024.11.0-py3-none-any.whl (1.2 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m118.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hDownloading packaging-24.2-py3-none-any.whl (65 kB)\n", - "Downloading pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.1 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.1/13.1 MB\u001b[0m \u001b[31m231.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hUsing cached tzdata-2024.2-py2.py3-none-any.whl (346 kB)\n", - "Installing collected packages: tzdata, packaging, pandas, xarray\n", - " Attempting uninstall: packaging\n", - " Found existing installation: packaging 21.3\n", - " Uninstalling packaging-21.3:\n", - " Successfully uninstalled packaging-21.3\n", - " Attempting uninstall: pandas\n", - " Found existing installation: pandas 1.5.3\n", - " Uninstalling pandas-1.5.3:\n", - " Successfully uninstalled pandas-1.5.3\n", - "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "autovizwidget 0.21.0 requires pandas<2.0.0,>=0.20.1, but you have pandas 2.2.3 which is incompatible.\n", - "hdijupyterutils 0.21.0 requires pandas<2.0.0,>=0.17.1, but you have pandas 2.2.3 which is incompatible.\n", - "sparkmagic 0.21.0 requires pandas<2.0.0,>=0.17.1, but you have pandas 2.2.3 which is incompatible.\n", - "sphinx 8.1.3 requires docutils<0.22,>=0.20, but you have docutils 0.16 which is incompatible.\u001b[0m\u001b[31m\n", - "\u001b[0mSuccessfully installed packaging-24.2 pandas-2.2.3 tzdata-2024.2 xarray-2024.11.0\n", - "Collecting geoalchemy2\n", - " Downloading GeoAlchemy2-0.16.0-py3-none-any.whl.metadata (2.1 kB)\n", - "Requirement already satisfied: SQLAlchemy>=1.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (2.0.36)\n", - "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from geoalchemy2) (24.2)\n", - "Requirement already satisfied: typing-extensions>=4.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (4.12.2)\n", - "Requirement already satisfied: greenlet!=0.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (3.1.1)\n", - "Downloading GeoAlchemy2-0.16.0-py3-none-any.whl (74 kB)\n", - "Installing collected packages: geoalchemy2\n", - "Successfully installed geoalchemy2-0.16.0\n", - "Collecting rioxarray\n", - " Downloading rioxarray-0.18.1-py3-none-any.whl.metadata (5.4 kB)\n", - "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (24.2)\n", - "Collecting rasterio>=1.3.7 (from rioxarray)\n", - " Downloading rasterio-1.4.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (9.1 kB)\n", - "Requirement already satisfied: xarray>=2024.7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (2024.11.0)\n", - "Requirement already satisfied: pyproj>=3.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (3.7.0)\n", - "Requirement already satisfied: numpy>=1.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rioxarray) (1.26.4)\n", - "Requirement already satisfied: certifi in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pyproj>=3.3->rioxarray) (2024.8.30)\n", - "Collecting affine (from rasterio>=1.3.7->rioxarray)\n", - " Downloading affine-2.4.0-py3-none-any.whl.metadata (4.0 kB)\n", - "Requirement already satisfied: attrs in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (23.2.0)\n", - "Requirement already satisfied: click>=4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (8.1.7)\n", - "Collecting cligj>=0.5 (from rasterio>=1.3.7->rioxarray)\n", - " Downloading cligj-0.7.2-py3-none-any.whl.metadata (5.0 kB)\n", - "Collecting click-plugins (from rasterio>=1.3.7->rioxarray)\n", - " Downloading click_plugins-1.1.1-py2.py3-none-any.whl.metadata (6.4 kB)\n", - "Requirement already satisfied: pyparsing in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from rasterio>=1.3.7->rioxarray) (3.2.0)\n", - "Requirement already satisfied: pandas>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from xarray>=2024.7.0->rioxarray) (2.2.3)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2.9.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas>=2.1->xarray>=2024.7.0->rioxarray) (2024.2)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=2.1->xarray>=2024.7.0->rioxarray) (1.16.0)\n", - "Downloading rioxarray-0.18.1-py3-none-any.whl (61 kB)\n", - "Downloading rasterio-1.4.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (22.2 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m22.2/22.2 MB\u001b[0m \u001b[31m144.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hDownloading cligj-0.7.2-py3-none-any.whl (7.1 kB)\n", - "Downloading affine-2.4.0-py3-none-any.whl (15 kB)\n", - "Downloading click_plugins-1.1.1-py2.py3-none-any.whl (7.5 kB)\n", - "Installing collected packages: cligj, click-plugins, affine, rasterio, rioxarray\n", - "Successfully installed affine-2.4.0 click-plugins-1.1.1 cligj-0.7.2 rasterio-1.4.3 rioxarray-0.18.1\n", - "Requirement already satisfied: s3fs in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (0.4.2)\n", - "Requirement already satisfied: botocore>=1.12.91 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from s3fs) (1.35.53)\n", - "Requirement already satisfied: fsspec>=0.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from s3fs) (2024.10.0)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (1.0.1)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (2.9.0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore>=1.12.91->s3fs) (2.2.3)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil<3.0.0,>=2.1->botocore>=1.12.91->s3fs) (1.16.0)\n", - "Collecting python-dotenv\n", - " Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n", - "Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", - "Installing collected packages: python-dotenv\n", - "Successfully installed python-dotenv-1.0.1\n", - "Requirement already satisfied: botocore in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (1.35.53)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore) (1.0.1)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore) (2.9.0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore) (2.2.3)\n", - "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil<3.0.0,>=2.1->botocore) (1.16.0)\n", - "*** All loaded ***\n" - ] - } - ], + "outputs": [], "source": [ "# Cell to manually pip reload a packages that the Jupyter engine not retained\n", "!pip install numpy\n", @@ -169,34 +45,13 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "5b656259", "metadata": { "scrolled": true, "tags": [] }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "option set\n", - "imports loaded\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import codecs\n", @@ -242,21 +97,12 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "fae5ca0f-e582-4ff2-89b6-087285433a97", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "aws_keys are at /home/ec2-user/SageMaker/AWS_keys.env\n", - "aws_keys loaded\n" - ] - } - ], + "outputs": [], "source": [ "# Load AWS Keys\n", "from dotenv import load_dotenv\n", @@ -279,20 +125,12 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "e944eff3-6023-48f4-9f57-27304a240447", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Global Variables loaded\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# we won't load this into any tables at this time\n", @@ -1062,78 +900,11 @@ "

7 - Recreate derived.usgs_elev_table

" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "c855b07d-ef8b-4dc6-94a7-e6d0f2faf279", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cb366d43-9aed-4487-841c-9888d701f6fb", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a22955a9-eb15-4065-9825-ccb964cc9c66", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aeb75842-ede9-4ea8-8fac-43b21b27de14", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d6a7842-2a24-4c5b-9fab-20ae3355c387", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ff2b74e2-bde9-43bc-8548-e6277e439ecc", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee4bdcd7-ce35-44e4-affa-64f827e47399", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c85a0d4a-4af1-495b-b837-780a485c8a95", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": null, "id": "2dd0c144-5a20-4899-b946-a6dd9ae49b53", "metadata": { - "jupyter": { - "source_hidden": true - }, "scrolled": true, "tags": [] }, @@ -1208,9 +979,6 @@ "execution_count": null, "id": "33026eb6-fbc7-47f7-a44b-f9cea8f0911b", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1371,9 +1139,6 @@ "execution_count": null, "id": "329dae61-d69f-4865-bdb6-e99a7442b0be", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1485,9 +1250,6 @@ "execution_count": null, "id": "5cc764e4-7e67-4b51-a100-6021e31416cd", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1526,9 +1288,6 @@ "execution_count": null, "id": "74f21c03-ca36-402c-a69b-396162720f66", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1554,9 +1313,6 @@ "execution_count": null, "id": "461f20b4-8442-4c0e-9114-ea62764cd789", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1577,9 +1333,6 @@ "execution_count": null, "id": "b752fce5-c8d7-4064-92d7-842c22d1723e", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1636,9 +1389,6 @@ "execution_count": null, "id": "4548d5e8-303e-44e0-a8a5-7b1947214b4e", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1670,9 +1420,6 @@ "execution_count": null, "id": "d1a77875-f569-4410-888c-b132714a7c9e", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1698,9 +1445,6 @@ "execution_count": null, "id": "c24f509a-90df-492a-a0d4-aa9eab8e7041", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1741,9 +1485,6 @@ "execution_count": null, "id": "b5639447-9a5a-4add-9c36-ebd945a5a8d0", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -1853,9 +1594,6 @@ "execution_count": null, "id": "06ca5581", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, "outputs": [], @@ -2034,23 +1772,12 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "37f37c81-d105-4c38-aa49-b4aef40a7543", "metadata": { - "jupyter": { - "source_hidden": true - }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "load_catfim_table function loaded\n" - ] - } - ], + "outputs": [], "source": [ "''' Function to load CatFIM data (for any flow / stage / library / sites but non public)'''\n", "\n", @@ -2292,77 +2019,15 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "95c55cd0", "metadata": { - "collapsed": true, "jupyter": { - "outputs_hidden": true, "source_hidden": true }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting of CatFIM data\n", - "Loading stage_based_catfim_library data\n", - "Dropping reference.stage_based_catfim_library table if it existed\n", - "\n", - ".. Downloading: s3://hydrovis-ti-deployment-us-east-1/fim/hand_4_5_11_1/qa_datasets/stage_based_catfim_library.csv\n", - ".. Downloading complete\n", - "File read. 34061 records to load\n", - "Chunk loading... into stage_based_catfim_library -- 1000 records at a time\n", - "\n", - " ... loading chunk 1 of 35\n", - " ... loading chunk 2 of 35\n", - " ... loading chunk 3 of 35\n", - " ... loading chunk 4 of 35\n", - " ... loading chunk 5 of 35\n", - " ... loading chunk 6 of 35\n", - " ... loading chunk 7 of 35\n", - " ... loading chunk 8 of 35\n", - " ... loading chunk 9 of 35\n", - " ... loading chunk 10 of 35\n", - " ... loading chunk 11 of 35\n", - " ... loading chunk 12 of 35\n", - " ... loading chunk 13 of 35\n", - " ... loading chunk 14 of 35\n", - " ... loading chunk 15 of 35\n", - " ... loading chunk 16 of 35\n", - " ... loading chunk 17 of 35\n", - " ... loading chunk 18 of 35\n", - " ... loading chunk 19 of 35\n", - " ... loading chunk 20 of 35\n", - " ... loading chunk 21 of 35\n", - " ... loading chunk 22 of 35\n", - " ... loading chunk 23 of 35\n", - " ... loading chunk 24 of 35\n", - " ... loading chunk 25 of 35\n", - " ... loading chunk 26 of 35\n", - " ... loading chunk 27 of 35\n", - " ... loading chunk 28 of 35\n", - " ... loading chunk 29 of 35\n", - " ... loading chunk 30 of 35\n", - " ... loading chunk 31 of 35\n", - " ... loading chunk 32 of 35\n", - " ... loading chunk 33 of 35\n", - " ... loading chunk 34 of 35\n", - " ... loading chunk 35 of 35\n", - "Loading stage_based_catfim_sites data\n", - "Dropping reference.stage_based_catfim_sites table if it existed\n", - "\n", - ".. Downloading: s3://hydrovis-ti-deployment-us-east-1/fim/hand_4_5_11_1/qa_datasets/stage_based_catfim_sites.csv\n", - ".. Downloading complete\n", - "File read. 4566 records to load\n", - "Loading data into stage_based_catfim_sites ...\n", - "\n", - "... duration was 0:13:42\n" - ] - } - ], + "outputs": [], "source": [ "\n", "print(\"Starting of CatFIM data\")\n", @@ -2463,7 +2128,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "fd38c00d-22ad-476e-ab05-cf293e5bbc15", "metadata": { "jupyter": { @@ -2471,23 +2136,7 @@ }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading CatFIM Public datasets (FIM 30)\n", - "\n", - "None\n", - "public fim_30 data load for stage_based_catfim is complete\n", - "\n", - "None\n", - "public fim_30 data load for stage_based_catfim_sites is complete\n", - "\n", - "... duration was 0:01:38\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Dec 2024: We don't use the flow based public db's\n", @@ -2539,8559 +2188,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "59c382de-d616-40f4-8aa5-e7cdcc8010de", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Bridge Centroids data load started\n", - "... loading from s3://hydrovis-ti-deployment-us-east-1/fim/hand_4_5_11_1/hand_datasets\n", - "\n", - "-------------------------------------------\n", - "Processing rec number 1 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 2 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 3 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 4 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 5 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 6 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 7 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 8 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 9 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 10 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01010011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01010011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 11 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 12 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 13 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 14 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 15 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 16 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 17 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 18 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 19 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 20 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 21 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 22 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 23 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 24 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 25 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01060001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01060001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 26 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01060002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 27 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01060003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01060003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 28 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01070001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01070001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 29 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01070002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01070002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 30 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01070003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01070003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 31 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01070004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01070004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 32 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01070005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01070005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 33 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01070006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01070006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 34 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 35 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 36 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 37 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 38 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 39 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 40 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 41 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 42 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 43 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 44 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 45 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 46 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 47 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01080207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01080207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 48 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01090001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01090001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 49 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01090002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01090002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 50 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01090003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01090003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 51 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01090004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01090004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 52 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01090005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01090005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 53 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01100001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01100001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 54 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01100002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01100002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 55 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01100003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01100003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 56 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01100004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01100004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 57 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01100005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01100005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 58 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/01100006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/01100006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 59 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 60 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 61 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 62 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 63 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 64 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 65 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02020007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02020007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 66 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02020008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02020008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 67 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02030101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02030101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 68 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02030102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02030102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 69 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02030103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02030103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 70 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02030104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02030104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 71 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02030105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02030105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 72 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02030201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02030201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 73 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02030202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02030202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 74 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02030203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02030203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 75 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 76 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 77 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 78 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 79 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 80 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 81 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 82 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 83 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 84 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 85 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 86 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 87 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 88 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 89 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 90 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02040304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02040304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 91 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 92 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 93 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 94 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 95 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 96 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 97 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 98 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 99 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 100 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 101 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 102 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 103 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 104 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 105 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 106 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 107 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 108 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 109 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02050306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02050306.gpkg\n", - "-------------------------------------------\n", - "Rec number 110 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 111 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02060002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 112 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02060003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02060003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 113 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02060004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02060004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 114 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02060005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02060005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 115 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02060006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02060006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 116 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 117 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 118 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 119 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 120 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 121 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 122 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 123 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 124 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 125 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 126 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02070011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02070011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 127 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 128 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 129 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 130 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 131 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 132 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 133 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 134 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 135 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 136 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 137 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 138 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 139 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 140 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 141 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 142 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 143 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 144 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/02080208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/02080208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 145 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 146 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 147 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 148 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 149 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 150 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 151 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 152 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 153 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 154 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 155 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 156 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03010205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03010205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 157 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 158 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 159 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 160 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 161 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 162 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 163 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 164 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 165 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 166 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 167 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03020302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03020302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 168 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 169 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 170 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03030004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03030004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 171 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03030005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03030005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 172 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03030006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03030006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 173 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03030007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03030007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 174 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 175 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 176 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 177 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 178 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 179 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 180 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 181 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 182 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 183 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 184 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 185 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 186 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03040208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03040208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 187 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 188 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 189 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 190 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 191 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 192 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 193 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 194 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 195 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 196 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 197 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 198 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050112/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050112.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 199 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 200 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 201 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 202 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 203 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 204 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 205 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 206 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 207 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 208 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03050210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03050210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 209 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 210 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 211 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 212 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 213 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 214 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 215 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 216 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 217 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 218 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 219 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 220 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 221 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 222 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03060204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03060204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 223 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 224 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 225 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 226 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 227 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 228 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 229 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 230 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 231 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 232 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 233 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 234 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03070205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03070205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 235 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03080101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03080101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 236 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03080102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03080102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 237 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03080103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03080103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 238 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03080201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03080201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 239 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03080202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03080202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 240 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03080203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03080203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 241 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03090101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03090101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 242 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03090102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03090102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 243 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03090103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03090103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 244 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03090201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03090201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 245 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03090202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03090202.gpkg\n", - "-------------------------------------------\n", - "Rec number 246 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 247 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03090204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03090204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 248 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03090205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03090205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 249 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03090206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03090206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 250 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 251 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 252 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 253 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 254 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 255 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 256 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 257 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 258 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 259 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 260 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03100208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03100208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 261 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 262 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 263 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 264 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 265 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 266 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 267 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 268 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 269 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03110206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03110206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 270 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03120001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03120001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 271 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03120002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03120002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 272 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03120003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03120003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 273 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 274 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 275 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 276 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 277 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 278 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 279 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 280 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 281 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 282 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 283 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 284 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 285 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 286 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03130014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03130014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 287 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 288 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 289 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 290 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 291 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 292 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 293 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 294 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 295 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 296 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 297 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 298 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 299 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 300 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 301 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03140305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03140305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 302 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 303 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 304 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 305 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 306 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 307 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 308 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 309 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 310 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 311 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 312 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 313 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 314 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 315 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03150204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03150204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 316 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 317 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 318 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 319 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 320 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 321 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 322 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 323 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 324 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 325 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 326 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 327 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160112/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160112.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 328 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160113/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160113.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 329 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 330 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 331 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 332 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 333 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03160205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03160205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 334 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 335 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 336 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 337 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 338 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 339 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 340 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 341 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 342 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03170009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03170009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 343 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03180001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03180001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 344 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03180002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03180002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 345 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03180003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03180003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 346 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03180004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03180004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 347 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/03180005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/03180005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 348 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04010101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04010101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 349 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04010102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04010102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 350 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04010201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04010201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 351 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04010202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04010202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 352 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04010301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04010301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 353 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04010302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04010302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 354 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04020101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04020101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 355 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04020102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04020102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 356 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04020103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04020103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 357 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04020104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04020104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 358 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04020105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04020105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 359 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04020201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04020201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 360 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04020202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04020202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 361 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04020203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04020203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 362 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 363 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 364 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 365 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 366 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 367 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 368 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 369 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 370 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 371 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 372 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 373 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030112/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030112.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 374 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030113/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030113.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 375 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030114/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030114.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 376 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 377 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 378 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 379 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04030204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04030204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 380 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 381 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 382 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04040003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04040003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 383 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 384 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 385 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 386 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 387 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04050005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04050005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 388 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04050006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04050006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 389 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04050007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04050007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 390 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04060101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04060101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 391 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04060102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04060102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 392 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04060103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04060103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 393 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04060104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04060104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 394 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04060105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04060105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 395 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04070003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04070003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 396 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04070004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04070004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 397 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04070005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04070005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 398 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04070006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04070006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 399 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04070007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04070007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 400 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 401 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 402 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 403 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 404 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 405 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 406 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 407 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 408 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 409 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04080206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04080206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 410 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04090001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04090001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 411 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04090002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04090002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 412 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04090003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04090003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 413 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04090004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04090004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 414 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04090006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04090006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 415 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04090007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04090007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 416 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 417 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 418 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 419 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 420 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 421 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 422 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 423 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 424 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 425 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 426 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 427 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 428 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04100013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04100013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 429 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04110001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04110001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 430 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04110002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04110002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 431 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04110003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04110003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 432 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04110004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04110004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 433 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04120101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04120101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 434 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04120102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04120102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 435 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04120103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04120103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 436 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04130001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04130001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 437 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04130002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04130002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 438 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04130003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04130003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 439 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04140101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04140101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 440 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04140102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04140102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 441 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04140201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04140201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 442 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04140202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04140202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 443 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04140203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04140203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 444 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04140301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04140301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 445 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04140302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04140302.gpkg\n", - "-------------------------------------------\n", - "Rec number 446 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 447 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04160002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04160002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 448 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04160003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04160003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 449 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04160004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04160004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 450 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04200001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04200001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 451 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04200002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04200002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 452 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04200003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04200003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 453 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04200004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04200004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 454 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04200005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04200005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 455 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04200006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04200006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 456 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04270101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04270101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 457 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04290001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04290001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 458 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04290002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04290002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 459 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04290003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04290003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 460 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04290004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04290004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 461 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04290005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04290005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 462 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04290006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04290006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 463 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04290007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04290007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 464 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04290008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04290008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 465 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 466 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 467 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 468 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 469 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 470 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 471 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 472 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 473 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 474 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/04300202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/04300202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 475 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 476 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 477 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 478 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 479 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 480 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 481 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 482 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 483 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05010009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05010009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 484 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 485 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 486 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 487 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 488 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 489 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 490 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 491 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 492 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 493 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 494 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 495 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 496 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 497 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 498 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 499 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05030204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05030204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 500 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 501 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 502 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05040003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05040003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 503 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05040004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05040004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 504 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05040005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05040005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 505 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05040006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05040006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 506 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 507 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 508 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 509 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 510 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 511 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 512 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 513 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 514 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05050009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05050009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 515 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05060001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05060001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 516 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05060002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 517 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05060003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05060003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 518 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05070101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05070101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 519 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05070102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05070102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 520 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05070201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05070201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 521 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05070202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05070202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 522 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05070203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05070203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 523 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05070204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05070204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 524 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05080001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05080001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 525 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05080002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05080002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 526 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05080003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05080003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 527 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05090101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05090101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 528 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05090102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05090102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 529 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05090103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05090103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 530 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05090104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05090104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 531 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05090201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05090201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 532 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05090202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05090202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 533 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05090203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05090203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 534 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05100101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05100101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 535 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05100102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05100102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 536 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05100201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05100201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 537 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05100202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05100202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 538 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05100203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05100203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 539 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05100204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05100204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 540 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05100205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05100205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 541 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05110001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05110001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 542 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05110002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05110002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 543 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05110003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05110003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 544 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05110004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05110004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 545 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05110005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05110005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 546 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05110006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05110006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 547 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 548 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 549 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 550 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 551 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 552 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 553 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 554 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 555 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 556 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 557 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 558 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120112/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120112.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 559 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120113/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120113.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 560 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120114/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120114.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 561 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120115/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120115.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 562 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 563 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 564 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 565 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 566 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 567 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 568 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 569 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 570 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05120209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05120209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 571 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 572 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 573 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 574 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 575 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 576 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 577 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 578 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 579 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 580 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 581 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 582 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 583 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 584 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05130206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05130206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 585 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 586 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 587 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 588 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 589 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 590 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 591 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 592 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 593 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 594 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/05140206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/05140206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 595 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 596 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 597 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 598 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 599 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 600 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 601 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 602 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 603 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 604 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 605 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 606 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 607 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 608 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 609 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 610 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06010208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06010208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 611 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 612 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 613 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 614 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 615 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 616 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 617 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 618 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06030004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06030004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 619 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06030005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06030005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 620 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06030006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06030006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 621 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 622 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 623 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06040003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06040003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 624 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06040004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06040004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 625 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06040005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06040005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 626 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/06040006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/06040006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 627 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 628 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 629 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 630 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 631 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 632 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 633 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 634 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 635 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 636 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 637 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 638 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 639 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 640 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 641 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07010207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07010207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 642 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 643 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 644 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 645 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 646 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 647 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 648 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 649 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 650 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 651 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 652 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 653 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07020012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07020012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 654 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 655 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 656 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 657 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07030004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07030004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 658 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07030005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07030005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 659 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 660 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 661 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07040003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07040003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 662 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07040004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07040004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 663 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07040005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07040005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 664 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07040006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07040006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 665 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07040007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07040007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 666 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07040008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07040008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 667 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 668 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 669 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 670 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 671 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07050005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07050005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 672 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07050006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07050006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 673 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07050007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07050007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 674 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07060001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07060001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 675 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07060002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 676 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07060003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07060003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 677 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07060004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07060004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 678 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07060005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07060005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 679 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07060006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07060006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 680 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07070001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07070001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 681 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07070002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07070002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 682 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07070003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07070003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 683 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07070004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07070004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 684 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07070005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07070005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 685 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07070006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07070006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 686 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 687 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 688 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 689 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 690 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 691 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 692 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 693 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 694 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 695 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 696 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 697 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 698 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 699 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 700 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 701 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07080209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07080209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 702 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07090001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07090001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 703 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07090002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07090002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 704 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07090003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07090003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 705 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07090004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07090004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 706 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07090005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07090005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 707 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07090006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07090006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 708 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07090007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07090007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 709 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 710 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 711 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 712 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 713 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 714 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 715 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 716 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 717 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07100009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07100009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 718 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 719 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 720 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 721 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 722 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 723 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 724 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 725 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 726 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07110009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07110009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 727 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07120001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07120001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 728 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07120002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07120002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 729 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07120003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07120003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 730 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07120004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07120004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 731 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07120005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07120005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 732 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07120006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07120006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 733 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07120007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07120007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 734 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 735 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 736 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 737 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 738 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 739 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 740 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 741 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 742 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 743 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 744 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 745 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07130012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07130012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 746 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 747 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 748 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 749 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 750 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 751 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 752 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 753 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 754 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 755 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 756 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 757 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/07140204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/07140204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 758 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 759 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 760 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 761 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 762 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 763 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 764 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 765 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 766 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 767 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 768 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 769 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010211/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010211.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 770 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08010300/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08010300.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 771 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 772 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 773 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 774 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 775 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 776 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 777 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 778 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 779 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 780 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020401/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020401.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 781 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08020402/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08020402.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 782 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 783 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 784 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 785 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 786 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 787 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 788 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 789 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 790 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 791 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08030209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08030209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 792 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 793 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 794 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 795 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 796 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 797 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 798 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 799 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 800 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 801 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 802 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 803 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 804 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 805 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 806 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 807 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08040306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08040306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 808 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 809 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 810 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 811 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08060100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08060100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 812 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08060201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08060201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 813 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08060202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08060202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 814 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08060203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08060203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 815 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08060204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08060204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 816 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08060205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08060205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 817 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08060206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08060206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 818 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08070100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08070100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 819 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08070201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08070201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 820 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08070202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08070202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 821 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08070203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08070203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 822 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08070204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08070204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 823 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08070205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08070205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 824 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08070300/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08070300.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 825 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 826 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 827 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 828 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 829 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 830 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 831 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 832 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 833 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08080206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08080206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 834 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08090100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08090100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 835 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08090201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08090201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 836 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08090202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08090202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 837 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08090203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08090203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 838 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08090301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08090301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 839 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/08090302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/08090302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 840 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 841 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 842 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 843 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09010005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 844 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09010006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09010006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 845 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09010007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09010007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 846 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09010008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09010008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 847 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 848 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 849 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 850 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 851 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 852 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 853 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 854 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 855 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 856 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 857 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 858 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 859 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 860 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 861 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 862 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 863 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 864 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 865 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 866 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 867 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020307/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020307.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 868 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020308/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020308.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 869 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020309/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020309.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 870 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020310/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020310.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 871 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020311/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020311.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 872 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020312/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020312.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 873 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020314/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020314.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 874 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020315/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020315.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 875 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09020316/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09020316.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 876 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 877 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 878 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 879 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09030005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09030005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 880 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09030006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09030006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 881 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09030007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09030007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 882 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09030008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09030008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 883 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09030009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09030009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 884 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 885 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/09040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/09040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 886 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 887 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 888 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 889 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 890 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 891 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 892 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10020007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10020007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 893 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10020008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10020008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 894 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 895 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 896 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 897 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 898 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 899 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 900 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 901 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 902 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 903 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10030205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10030205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 904 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 905 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 906 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 907 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 908 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 909 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 910 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 911 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 912 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 913 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 914 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10040205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10040205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 915 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 916 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 917 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 918 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 919 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 920 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 921 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 922 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 923 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 924 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 925 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 926 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 927 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 928 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 929 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10050016/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10050016.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 930 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10060001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10060001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 931 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10060002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 932 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10060003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10060003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 933 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10060004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10060004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 934 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10060005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10060005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 935 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10060006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10060006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 936 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10060007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10060007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 937 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10070001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10070001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 938 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10070002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10070002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 939 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10070003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10070003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 940 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10070004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10070004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 941 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10070005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10070005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 942 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10070006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10070006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 943 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10070007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10070007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 944 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10070008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10070008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 945 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 946 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 947 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 948 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 949 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 950 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 951 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 952 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 953 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 954 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 955 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 956 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 957 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 958 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 959 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 960 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10080016/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10080016.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 961 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 962 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 963 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 964 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 965 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 966 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 967 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 968 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 969 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 970 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 971 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 972 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10090210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10090210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 973 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10100001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10100001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 974 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10100002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10100002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 975 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10100003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10100003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 976 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10100004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10100004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 977 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10100005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10100005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 978 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10110101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10110101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 979 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10110102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10110102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 980 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10110201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10110201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 981 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10110202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10110202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 982 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10110203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10110203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 983 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10110204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10110204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 984 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10110205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10110205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 985 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 986 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 987 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 988 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 989 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 990 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 991 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 992 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 993 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 994 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 995 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 996 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120112/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120112.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 997 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120113/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120113.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 998 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 999 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 1000 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10120203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10120203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 1 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 2 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 3 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 4 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 5 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 6 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 7 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 8 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 9 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 10 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 11 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 12 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 13 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 14 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 15 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 16 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 17 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 18 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10130306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10130306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 19 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 20 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 21 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 22 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 23 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 24 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 25 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 26 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 27 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10140204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10140204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 28 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10150002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10150002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 29 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10150003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10150003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 30 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10150004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10150004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 31 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10150005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10150005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 32 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10150006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10150006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 33 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10150007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10150007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 34 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 35 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 36 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 37 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 38 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 39 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160006.gpkg\n", - "-------------------------------------------\n", - "Rec number 40 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 41 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 42 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 43 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10160011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10160011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 44 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10170101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10170101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 45 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10170102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10170102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 46 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10170103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10170103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 47 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10170104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10170104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 48 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10170201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10170201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 49 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10170202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10170202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 50 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10170203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10170203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 51 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10170204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10170204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 52 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 53 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 54 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 55 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 56 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 57 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 58 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 59 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 60 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 61 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 62 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 63 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 64 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 65 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10180014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10180014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 66 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 67 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 68 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 69 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 70 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 71 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 72 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 73 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 74 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 75 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 76 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 77 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 78 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 79 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 80 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 81 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190016/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190016.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 82 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190017/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190017.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 83 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10190018/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10190018.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 84 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10200101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10200101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 85 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10200102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10200102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 86 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10200103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10200103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 87 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10200201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10200201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 88 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10200202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10200202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 89 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10200203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10200203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 90 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 91 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 92 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 93 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 94 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 95 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 96 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 97 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 98 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 99 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10210010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10210010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 100 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10220001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10220001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 101 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10220002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10220002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 102 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10220003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10220003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 103 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10220004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10220004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 104 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10230001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10230001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 105 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10230002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10230002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 106 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10230003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10230003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 107 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10230004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10230004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 108 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10230005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10230005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 109 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10230006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10230006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 110 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10230007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10230007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 111 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 112 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 113 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 114 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 115 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 116 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 117 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 118 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 119 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 120 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 121 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 122 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 123 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10240013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10240013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 124 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 125 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 126 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 127 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 128 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 129 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 130 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 131 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 132 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 133 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 134 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 135 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 136 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 137 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 138 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 139 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250016/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250016.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 140 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10250017/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10250017.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 141 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 142 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 143 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 144 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 145 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 146 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 147 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 148 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 149 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 150 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 151 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 152 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 153 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 154 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 155 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10260015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10260015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 156 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 157 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 158 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 159 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 160 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 161 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 162 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 163 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 164 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 165 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 166 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10270207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10270207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 167 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10280101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10280101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 168 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10280102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10280102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 169 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10280103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10280103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 170 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10280201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10280201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 171 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10280202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10280202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 172 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10280203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10280203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 173 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 174 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 175 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 176 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 177 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 178 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 179 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 180 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 181 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 182 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 183 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 184 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 185 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 186 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10290203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10290203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 187 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10300101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10300101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 188 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10300102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10300102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 189 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10300103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10300103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 190 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10300104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10300104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 191 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/10300200/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/10300200.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 192 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 193 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 194 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 195 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 196 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 197 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 198 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 199 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 200 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 201 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 202 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 203 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 204 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 205 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11010014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11010014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 206 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 207 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 208 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 209 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 210 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 211 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 212 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 213 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 214 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 215 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 216 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 217 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 218 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11020013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11020013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 219 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 220 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 221 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 222 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 223 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 224 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 225 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 226 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 227 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 228 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 229 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 230 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 231 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 232 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 233 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 234 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030016/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030016.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 235 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030017/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030017.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 236 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11030018/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11030018.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 237 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 238 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 239 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11040003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11040003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 240 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11040004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11040004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 241 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11040005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11040005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 242 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11040006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11040006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 243 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11040007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11040007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 244 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11040008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11040008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 245 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 246 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 247 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 248 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11060001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11060001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 249 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11060002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 250 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11060003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11060003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 251 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11060004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11060004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 252 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11060005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11060005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 253 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11060006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11060006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 254 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 255 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 256 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 257 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 258 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 259 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 260 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 261 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 262 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 263 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 264 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 265 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 266 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 267 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 268 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 269 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11070209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11070209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 270 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11080001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11080001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 271 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11080002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11080002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 272 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11080003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11080003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 273 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11080004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11080004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 274 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11080005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11080005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 275 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11080006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11080006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 276 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11080007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11080007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 277 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11080008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11080008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 278 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 279 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 280 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 281 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 282 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 283 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 284 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 285 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 286 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 287 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11090204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11090204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 288 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 289 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 290 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 291 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 292 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 293 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 294 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 295 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 296 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 297 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11100303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11100303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 298 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 299 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 300 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 301 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 302 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 303 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 304 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 305 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 306 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 307 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 308 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 309 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11110207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11110207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 310 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 311 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 312 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 313 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 314 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 315 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 316 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 317 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 318 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 319 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 320 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11120304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11120304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 321 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 322 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 323 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 324 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 325 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 326 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 327 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 328 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 329 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 330 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 331 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 332 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 333 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 334 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 335 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 336 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 337 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 338 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 339 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11130304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11130304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 340 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 341 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 342 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 343 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 344 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 345 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 346 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 347 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 348 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 349 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 350 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 351 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 352 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 353 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 354 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 355 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 356 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 357 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 358 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 359 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 360 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 361 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 362 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 363 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 364 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/11140307/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/11140307.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 365 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12010001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12010001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 366 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 367 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 368 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 369 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12010005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 370 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 371 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 372 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 373 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 374 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 375 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 376 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12020007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12020007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 377 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 378 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 379 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 380 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 381 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 382 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 383 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 384 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 385 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 386 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 387 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 388 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12030203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12030203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 389 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 390 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 391 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 392 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 393 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 394 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 395 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 396 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 397 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12040205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12040205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 398 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 399 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 400 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 401 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 402 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12050005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12050005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 403 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12050006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12050006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 404 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12050007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12050007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 405 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 406 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 407 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 408 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 409 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 410 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 411 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 412 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 413 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12060204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12060204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 414 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 415 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 416 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 417 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 418 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 419 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 420 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 421 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 422 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12070205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12070205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 423 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12080001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12080001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 424 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12080002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12080002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 425 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12080003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12080003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 426 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12080004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12080004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 427 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12080005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12080005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 428 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12080006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12080006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 429 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12080007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12080007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 430 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12080008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12080008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 431 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 432 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 433 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 434 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 435 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 436 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 437 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 438 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 439 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 440 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 441 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 442 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 443 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 444 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 445 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 446 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 447 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 448 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 449 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090401/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090401.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 450 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12090402/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12090402.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 451 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 452 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 453 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 454 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 455 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 456 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 457 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 458 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 459 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 460 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 461 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100401/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100401.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 462 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100402/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100402.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 463 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100403/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100403.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 464 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100404/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100404.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 465 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100405/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100405.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 466 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100406/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100406.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 467 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12100407/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12100407.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 468 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110101.gpkg\n", - "-------------------------------------------\n", - "Rec number 469 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 470 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 471 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 472 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 473 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 474 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 475 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 476 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 477 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 478 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 479 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 480 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110202.gpkg\n", - "-------------------------------------------\n", - "Rec number 481 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 482 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 483 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 484 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110206.gpkg\n", - "-------------------------------------------\n", - "Rec number 485 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 486 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/12110208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/12110208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 487 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13010001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13010001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 488 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 489 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 490 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 491 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13010005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 492 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 493 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 494 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 495 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 496 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 497 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 498 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020205.gpkg\n", - "-------------------------------------------\n", - "Rec number 499 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 500 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 501 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 502 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020209.gpkg\n", - "-------------------------------------------\n", - "Rec number 503 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 504 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13020211/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13020211.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 505 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13030101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13030101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 506 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13030102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13030102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 507 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13030103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13030103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 508 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13030201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13030201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 509 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13030202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13030202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 510 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 511 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 512 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 513 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 514 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 515 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 516 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 517 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 518 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 519 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 520 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 521 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040211/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040211.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 522 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040212/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040212.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 523 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 524 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 525 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13040303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13040303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 526 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13050001.gpkg\n", - "-------------------------------------------\n", - "Rec number 527 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 528 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 529 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 530 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 531 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 532 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 533 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 534 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 535 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 536 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 537 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 538 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 539 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 540 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13060011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13060011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 541 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 542 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 543 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 544 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 545 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 546 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 547 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 548 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 549 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 550 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 551 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 552 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13070012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13070012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 553 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13080001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13080001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 554 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13080002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13080002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 555 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13080003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13080003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 556 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13090001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13090001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 557 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/13090002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/13090002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 558 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14010001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14010001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 559 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 560 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 561 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 562 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14010005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 563 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 564 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 565 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 566 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 567 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 568 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 569 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 570 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 571 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 572 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14030004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14030004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 573 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14030005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14030005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 574 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 575 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 576 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 577 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 578 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 579 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 580 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 581 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 582 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 583 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14040200/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14040200.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 584 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 585 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 586 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 587 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 588 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14050005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14050005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 589 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14050006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14050006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 590 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14050007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14050007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 591 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14060003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14060003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 592 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14060004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14060004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 593 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14060005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14060005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 594 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14060006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14060006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 595 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14060007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14060007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 596 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14060008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14060008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 597 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14060009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14060009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 598 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14060010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14060010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 599 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14070001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14070001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 600 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14070002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14070002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 601 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14070003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14070003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 602 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14070004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14070004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 603 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14070005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14070005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 604 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14070006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14070006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 605 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14070007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14070007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 606 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 607 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 608 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 609 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 610 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 611 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 612 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 613 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 614 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 615 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 616 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 617 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/14080205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/14080205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 618 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 619 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 620 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 621 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 622 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010005.gpkg\n", - "-------------------------------------------\n", - "Rec number 623 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 624 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 625 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 626 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 627 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 628 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 629 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 630 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 631 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 632 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15010015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15010015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 633 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 634 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 635 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 636 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 637 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 638 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 639 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 640 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 641 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 642 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 643 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 644 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 645 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 646 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 647 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 648 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020016/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020016.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 649 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020017/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020017.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 650 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15020018/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15020018.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 651 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 652 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 653 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 654 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 655 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 656 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 657 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 658 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 659 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 660 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 661 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 662 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15030204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15030204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 663 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 664 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 665 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15040003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15040003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 666 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15040004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15040004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 667 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15040005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15040005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 668 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15040006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15040006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 669 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15040007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15040007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 670 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 671 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 672 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 673 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 674 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 675 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 676 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 677 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 678 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 679 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15050306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15050306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 680 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 681 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 682 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 683 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 684 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 685 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 686 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 687 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 688 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15060203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15060203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 689 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15070101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15070101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 690 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15070102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15070102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 691 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15070103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15070103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 692 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15070104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15070104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 693 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15070201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15070201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 694 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15070202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15070202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 695 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15070203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15070203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 696 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15080101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15080101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 697 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15080102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15080102.gpkg\n", - "-------------------------------------------\n", - "Rec number 698 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 699 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 700 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15080301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15080301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 701 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15080302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15080302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 702 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/15080303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/15080303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 703 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16010101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16010101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 704 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16010102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16010102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 705 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16010201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16010201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 706 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16010202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16010202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 707 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16010203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16010203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 708 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16010204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16010204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 709 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 710 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 711 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 712 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 713 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 714 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 715 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020301.gpkg\n", - "-------------------------------------------\n", - "Rec number 716 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 717 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 718 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 719 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 720 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 721 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020307/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020307.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 722 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020308/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020308.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 723 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020309/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020309.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 724 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16020310/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16020310.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 725 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 726 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 727 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 728 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16030004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16030004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 729 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16030005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16030005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 730 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16030006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16030006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 731 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16030007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16030007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 732 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16030008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16030008.gpkg\n", - "-------------------------------------------\n", - "Rec number 733 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 734 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 735 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 736 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 737 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 738 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 739 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 740 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 741 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 742 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 743 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 744 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 745 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 746 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 747 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16040205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16040205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 748 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 749 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 750 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 751 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 752 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 753 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 754 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 755 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 756 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 757 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 758 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16050304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16050304.gpkg\n", - "-------------------------------------------\n", - "Rec number 759 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 760 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16060002.gpkg\n", - "-------------------------------------------\n", - "Rec number 761 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 762 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 763 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 764 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 765 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16060007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16060007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 766 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16060008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16060008.gpkg\n", - "-------------------------------------------\n", - "Rec number 767 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 768 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16060010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16060010.gpkg\n", - "-------------------------------------------\n", - "Rec number 769 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 770 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 771 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 772 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 773 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/16060015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/16060015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 774 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 775 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 776 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 777 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 778 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 779 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 780 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 781 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 782 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 783 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 784 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 785 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 786 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 787 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 788 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 789 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 790 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010211/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010211.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 791 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010212/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010212.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 792 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010213/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010213.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 793 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010214/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010214.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 794 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010215/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010215.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 795 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010216/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010216.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 796 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 797 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 798 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 799 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 800 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 801 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 802 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010307/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010307.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 803 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17010308/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17010308.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 804 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 805 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 806 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 807 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 808 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 809 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 810 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 811 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 812 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 813 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 814 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 815 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 816 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 817 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 818 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 819 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17020016/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17020016.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 820 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 821 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 822 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 823 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 824 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 825 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 826 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 827 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 828 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 829 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 830 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 831 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 832 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 833 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 834 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 835 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 836 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 837 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 838 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040211/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040211.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 839 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040212/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040212.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 840 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040213/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040213.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 841 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040214/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040214.gpkg\n", - "-------------------------------------------\n", - "Rec number 842 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 843 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040216/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040216.gpkg\n", - "-------------------------------------------\n", - "Rec number 844 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 845 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040218/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040218.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 846 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040219/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040219.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 847 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040220/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040220.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 848 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17040221/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17040221.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 849 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 850 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 851 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 852 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 853 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 854 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 855 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 856 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050108.gpkg\n", - "-------------------------------------------\n", - "Rec number 857 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 858 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 859 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 860 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050112/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050112.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 861 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050113/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050113.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 862 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050114/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050114.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 863 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050115/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050115.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 864 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050116/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050116.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 865 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050117/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050117.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 866 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050118/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050118.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 867 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050119/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050119.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 868 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050120/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050120.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 869 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050121/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050121.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 870 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050122/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050122.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 871 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050123/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050123.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 872 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050124/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050124.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 873 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 874 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 875 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17050203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17050203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 876 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 877 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 878 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 879 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 880 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 881 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 882 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 883 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 884 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 885 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 886 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 887 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 888 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 889 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 890 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 891 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 892 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 893 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 894 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 895 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 896 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 897 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 898 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 899 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 900 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 901 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 902 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060307/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060307.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 903 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17060308/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17060308.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 904 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 905 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 906 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 907 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 908 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 909 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 910 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 911 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 912 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 913 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 914 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 915 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 916 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 917 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 918 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 919 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 920 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17070307/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17070307.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 921 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17080001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17080001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 922 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17080002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17080002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 923 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17080003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17080003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 924 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17080004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17080004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 925 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17080005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17080005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 926 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17080006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17080006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 927 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 928 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 929 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 930 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 931 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 932 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 933 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 934 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 935 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 936 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 937 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 938 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17090012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17090012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 939 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 940 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 941 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 942 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 943 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 944 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 945 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 946 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 947 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 948 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 949 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 950 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 951 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 952 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 953 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 954 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 955 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 956 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100305.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 957 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100306/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100306.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 958 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100307/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100307.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 959 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100308/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100308.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 960 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100309/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100309.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 961 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100310/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100310.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 962 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100311/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100311.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 963 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17100312/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17100312.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 964 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 965 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 966 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 967 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 968 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 969 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 970 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 971 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 972 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 973 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 974 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 975 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 976 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 977 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 978 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 979 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110016/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110016.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 980 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110017/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110017.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 981 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110018/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110018.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 982 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110019/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110019.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 983 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110020/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110020.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 984 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17110021/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17110021.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 985 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17120001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17120001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 986 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17120002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17120002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 987 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17120003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17120003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 988 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17120004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17120004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 989 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17120005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17120005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 990 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17120006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17120006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 991 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17120007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17120007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 992 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/17120008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/17120008.gpkg\n", - "-------------------------------------------\n", - "Rec number 993 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 994 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 995 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 996 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 997 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 998 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 999 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 1000 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 1 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010108/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010108.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 2 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010109/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010109.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 3 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010110/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010110.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 4 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 5 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 6 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 7 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 8 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 9 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 10 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 11 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 12 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010209/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010209.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 13 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010210/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010210.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 14 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010211/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010211.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 15 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18010212/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18010212.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 16 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 17 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 18 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 19 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 20 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 21 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 22 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020111/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020111.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 23 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020115/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020115.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 24 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020116/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020116.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 25 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020121/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020121.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 26 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020122/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020122.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 27 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020123/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020123.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 28 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020125/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020125.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 29 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020126/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020126.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 30 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020128/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020128.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 31 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020129/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020129.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 32 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020151/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020151.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 33 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020152/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020152.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 34 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020153/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020153.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 35 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020154/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020154.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 36 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020155/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020155.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 37 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020156/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020156.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 38 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020157/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020157.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 39 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020158/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020158.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 40 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020159/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020159.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 41 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020161/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020161.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 42 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020162/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020162.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 43 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18020163/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18020163.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 44 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 45 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 46 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 47 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 48 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 49 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 50 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 51 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 52 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 53 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18030012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18030012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 54 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 55 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 56 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 57 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 58 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 59 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 60 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 61 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 62 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040011/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040011.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 63 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040012/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040012.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 64 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040013.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 65 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040014/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040014.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 66 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18040051/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18040051.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 67 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18050001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18050001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 68 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18050002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18050002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 69 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18050003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18050003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 70 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18050004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18050004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 71 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18050005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18050005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 72 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18050006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18050006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 73 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060002.gpkg\n", - "-------------------------------------------\n", - "Rec number 74 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 75 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 76 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 77 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060006/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060006.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 78 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060007/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060007.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 79 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 80 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060009/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060009.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 81 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060010/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060010.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 82 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060013/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060013.gpkg\n", - "-------------------------------------------\n", - "Rec number 83 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 84 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18060015/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18060015.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 85 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 86 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 87 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 88 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 89 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070105/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070105.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 90 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070106/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070106.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 91 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070107/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070107.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 92 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 93 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 94 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 95 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 96 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 97 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 98 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070303/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070303.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 99 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070304/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070304.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 100 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18070305/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18070305.gpkg\n", - "-------------------------------------------\n", - "Rec number 101 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 102 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 103 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18080003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18080003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 104 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 105 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 106 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090103.gpkg\n", - "-------------------------------------------\n", - "Rec number 107 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 108 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 109 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 110 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 111 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090205/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090205.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 112 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090206/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090206.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 113 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090207/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090207.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 114 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18090208/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18090208.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 115 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18100100/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18100100.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 116 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18100201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18100201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 117 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18100202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18100202.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 118 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18100203/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18100203.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 119 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/18100204/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/18100204.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 120 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020101/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020101.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 121 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020102/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020102.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 122 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020103/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020103.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 123 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020104/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020104.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 124 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020201/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020201.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 125 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020202/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020202.gpkg\n", - "-------------------------------------------\n", - "Rec number 126 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 127 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020301/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020301.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 128 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020302/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020302.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 129 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020401/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020401.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 130 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020402/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020402.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 131 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020501/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020501.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 132 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020502/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020502.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 133 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020503/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020503.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 134 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020504/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020504.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 135 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020505/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020505.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 136 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/19020601/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/19020601.gpkg\n", - "-------------------------------------------\n", - "Rec number 137 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Rec number 138 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 139 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/20010000/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/20010000.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 140 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/20020000/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/20020000.gpkg\n", - "-------------------------------------------\n", - "Rec number 141 not found and skipped (and it is fine)\n", - "-------------------------------------------\n", - "Processing rec number 142 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/20040000/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/20040000.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 143 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/20050000/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/20050000.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 144 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/20060000/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/20060000.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 145 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/20070000/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/20070000.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 146 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/20080000/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/20080000.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 147 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/21010002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/21010002.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 148 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/21010003/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/21010003.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 149 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/21010004/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/21010004.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 150 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/21010005/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/21010005.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 151 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/21010008/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/21010008.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 152 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/21020001/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/21020001.gpkg\n", - "-------------------------------------------\n", - "Processing rec number 153 of appx 2,150\n", - "... Loading /fim/hand_4_5_11_1/hand_datasets/21020002/osm_bridge_centroids.gpkg\n", - "........ to /home/ec2-user/SageMaker/temp_files/bridges/21020002.gpkg\n", - "============================\n", - "Removing all temp files\n", - "\n", - "Bridge Centroids data load completed\n", - "... duration was 0:09:04\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Has appx 2,150 HUCs to process, but this section goes quickly.\n", @@ -11237,12 +2337,13 @@ "outputs": [], "source": [ "sql = \"\"\"\n", - "TRUNCATE TABLE fim_cache.hand_hydrotable_cached;\n", - "TRUNCATE TABLE fim_cache.hand_hydrotable_cached_max;\n", - "TRUNCATE TABLE fim_cache.hand_hydrotable_cached_geo;\n", - "TRUNCATE TABLE fim_cache.hand_hydrotable_cached_zero_stage;\n", + "TRUNCATE TABLE handfim_cache.hydrotable_cached;\n", + "TRUNCATE TABLE handfim_cache.hydrotable_cached_geo;\n", + "TRUNCATE TABLE handfim_cache.hydrotable_cached_max;\n", + "TRUNCATE TABLE handfim_cache.hydrotable_cached_zero_stage;\n", "\"\"\"\n", - "sf.execute_sql(sql)" + "sf.execute_sql(sql)\n", + "print(\"caches cleared\")" ] }, { From 3166811c5e6f9185fedd3937478ff3eb9f43f47a Mon Sep 17 00:00:00 2001 From: RobHanna-NOAA Date: Tue, 10 Dec 2024 16:16:43 -0700 Subject: [PATCH 16/18] Fixed new db name for catfim_public --- .../reference/static_stage_based_catfim.mapx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Core/LAMBDA/viz_functions/viz_publish_service/services/reference/static_stage_based_catfim.mapx b/Core/LAMBDA/viz_functions/viz_publish_service/services/reference/static_stage_based_catfim.mapx index 8826631b..5b45eb92 100644 --- a/Core/LAMBDA/viz_functions/viz_publish_service/services/reference/static_stage_based_catfim.mapx +++ b/Core/LAMBDA/viz_functions/viz_publish_service/services/reference/static_stage_based_catfim.mapx @@ -688,7 +688,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%nws_flood_categorical_hand_fim_sites_1_2_1_2", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,wrds_timestamp,nrldb_timestamp,nwis_timestamp,metadata_sources,ahps_lid,usgs_gage,nwm_seg,identifiers_goes_id,identifiers_env_can_gage_id,nws_data_name,nws_data_wfo,nws_data_rfc,nws_data_geo_rfc,nws_data_latitude,nws_data_longitude,nws_data_map_link,nws_data_horizontal_datum_name,nws_data_state,nws_data_county,nws_data_county_code,nws_data_huc,nws_data_hsa,nws_data_zero_datum,nws_data_vertical_datum_name,nws_data_rfc_forecast_point,nws_data_rfc_defined_fcst_point,nws_data_riverpoint,usgs_data_name,usgs_data_geo_rfc,usgs_data_map_link,usgs_data_coord_accuracy_code,usgs_data_latlon_datum_name,usgs_data_coord_method_code,usgs_data_state,usgs_data_huc,usgs_data_site_type,usgs_data_altitude,usgs_data_alt_accuracy_code,usgs_data_alt_datum_code,usgs_data_alt_method_code,usgs_data_drainage_area,usgs_data_drainage_area_units,usgs_data_contrib_drainage_area,usgs_data_active,usgs_data_gages_ii_reference,nwm_feature_data_stream_order,nws_preferred_name,nws_preferred_latlon_datum_name,nws_preferred_state,nws_preferred_huc,usgs_preferred_name,usgs_preferred_latlon_datum_name,usgs_preferred_state,assigned_crs,huc8,name,states,mapped,status,geom from hydrovis.reference.stage_based_catfim_sites_public", + "sqlQuery" : "select oid,wrds_timestamp,nrldb_timestamp,nwis_timestamp,metadata_sources,ahps_lid,usgs_gage,nwm_seg,identifiers_goes_id,identifiers_env_can_gage_id,nws_data_name,nws_data_wfo,nws_data_rfc,nws_data_geo_rfc,nws_data_latitude,nws_data_longitude,nws_data_map_link,nws_data_horizontal_datum_name,nws_data_state,nws_data_county,nws_data_county_code,nws_data_huc,nws_data_hsa,nws_data_zero_datum,nws_data_vertical_datum_name,nws_data_rfc_forecast_point,nws_data_rfc_defined_fcst_point,nws_data_riverpoint,usgs_data_name,usgs_data_geo_rfc,usgs_data_map_link,usgs_data_coord_accuracy_code,usgs_data_latlon_datum_name,usgs_data_coord_method_code,usgs_data_state,usgs_data_huc,usgs_data_site_type,usgs_data_altitude,usgs_data_alt_accuracy_code,usgs_data_alt_datum_code,usgs_data_alt_method_code,usgs_data_drainage_area,usgs_data_drainage_area_units,usgs_data_contrib_drainage_area,usgs_data_active,usgs_data_gages_ii_reference,nwm_feature_data_stream_order,nws_preferred_name,nws_preferred_latlon_datum_name,nws_preferred_state,nws_preferred_huc,usgs_preferred_name,usgs_preferred_latlon_datum_name,usgs_preferred_state,assigned_crs,huc8,name,states,mapped,status,geom from hydrovis.reference.stage_based_catfim_sites_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -1876,7 +1876,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim_1", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,huc8,interval_stage,name,wfo,rfc,state,county,stage,stage_uni,s_src,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,huc8,interval_stage,name,wfo,rfc,state,county,stage,stage_uni,s_src,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -2619,7 +2619,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -3318,7 +3318,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim_1", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,huc8,interval_stage,name,wfo,rfc,state,county,stage,stage_uni,s_src,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,huc8,interval_stage,name,wfo,rfc,state,county,stage,stage_uni,s_src,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -4061,7 +4061,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -4760,7 +4760,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim_1", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,huc8,interval_stage,name,wfo,rfc,state,county,stage,stage_uni,s_src,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,huc8,interval_stage,name,wfo,rfc,state,county,stage,stage_uni,s_src,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -5503,7 +5503,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -6193,7 +6193,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim_1", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,huc8,interval_stage,name,wfo,rfc,state,county,stage,stage_uni,s_src,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,huc8,interval_stage,name,wfo,rfc,state,county,stage,stage_uni,s_src,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -6936,7 +6936,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, @@ -7789,7 +7789,7 @@ "workspaceFactory" : "SDE", "dataset" : "hydrovis.reference.%stage_based_catfim", "datasetType" : "esriDTFeatureClass", - "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_public", + "sqlQuery" : "select oid,ahps_lid,magnitude,fim_version,model_version,huc8,interval_stage,name,wfo,rfc,state,county,q,q_uni,q_src,stage,stage_uni,s_src,wrds_time,nrldb_time,nwis_time,lat,lon,dtm_adj_ft,dadj_w_ft,dadj_w_m,lid_alt_ft,lid_alt_m,viz,geom,fim_version,model_version from hydrovis.reference.stage_based_catfim_fim_30", "srid" : "3857", "spatialReference" : { "wkid" : 102100, From 86534dcf5359f272264e925b684cdc73cfc28d23 Mon Sep 17 00:00:00 2001 From: RobHanna-NOAA Date: Mon, 16 Dec 2024 09:22:57 -0700 Subject: [PATCH 17/18] small script tweaks --- .../FIM_data_loads/hand_4_5_11_1_update.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 3390df7f..56f6de16 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -917,7 +917,7 @@ "\n", "uet_usecols = ['location_id', 'HydroID', 'dem_adj_elevation', 'nws_lid', 'levpa_id']\n", "\n", - "viz_db_engine = sf.get_db_engine('viz')\n", + "db_engine = sf.get_db_engine('viz')\n", "s3_client = boto3.client(\"s3\")\n", "paginator = s3_client.get_paginator('list_objects')\n", "operation_parameters = {'Bucket': FIM_BUCKET,\n", @@ -945,7 +945,7 @@ " # uet_df['fim_version'] = PUBLIC_FIM_VERSION\n", " uet_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", " uet_df.to_sql(\n", - " con=viz_db_engine,\n", + " con=db_engine,\n", " dtype={\n", " \"location_id\": Text(),\n", " \"nws_data_huc\": Text()\n", @@ -1157,7 +1157,7 @@ "start_dt = dt.datetime.now()\n", "print(f\"Started: {dt.datetime.now().strftime('%m/%d/%Y, %H:%M:%S')}\")\n", "\n", - "viz_db_engine = sf.get_db_engine('viz')\n", + "db_engine = sf.get_db_engine('viz')\n", "sf.execute_sql('DROP TABLE IF EXISTS derived.hydrotable;')\n", "sql = '''\n", "SELECT distinct LPAD(huc8::text, 8, '0') as huc8 FROM derived.featureid_huc_crosswalk WHERE huc8 is not null;\n", @@ -1220,7 +1220,7 @@ " # print(\"... saving dataframe to db\")\n", " ht_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", " ht_df.to_sql(\n", - " con=viz_db_engine,\n", + " con=db_engine,\n", " schema='derived',\n", " name='hydrotable',\n", " index=False,\n", @@ -2217,7 +2217,7 @@ " region_name='us-east-1')\n", "\n", "s3_client = s3_session.client('s3')\n", - "viz_db_engine = sf.get_db_engine('egis')\n", + "db_engine = sf.get_db_engine('egis')\n", "\n", "paginator = s3_client.get_paginator('list_objects_v2')\n", "operation_parameters = {'Bucket': FIM_BUCKET,\n", @@ -2282,7 +2282,7 @@ " reproj_bridges_df[COLUMN_NAME_MODEL_VERSION] = HAND_MODEL_VERSION\n", "\n", " reproj_bridges_df.to_postgis(\n", - " con=viz_db_engine,\n", + " con=db_engine,\n", " schema='reference',\n", " name='bridge_centroids',\n", " index=True,\n", From 19e7f0fbb0457974eb1034b4c1700cdc88d09883 Mon Sep 17 00:00:00 2001 From: RobHanna-NOAA Date: Mon, 16 Dec 2024 09:43:57 -0700 Subject: [PATCH 18/18] typo fix --- Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb index 56f6de16..01d72a47 100644 --- a/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb +++ b/Core/Manual_Workflows/FIM_data_loads/hand_4_5_11_1_update.ipynb @@ -2199,7 +2199,7 @@ "print(\"Bridge Centroids data load started\")\n", "start_dt = dt.datetime.now()\n", "\n", - "sf.execute_sql('DROP TABLE IF EXISTS reference.bridge_centroidS;')\n", + "sf.execute_sql('DROP TABLE IF EXISTS reference.bridge_centroids;')\n", "\n", "# Set temp folder and file path for each gpkg to download too in order to laod\n", "# In later versions, we will have FIM-dev to create them as csv's but for now\n",