Skip to content

Commit

Permalink
Merge pull request #12 from datopian/fix/latest-dataset-and-faq
Browse files Browse the repository at this point in the history 
latest dataset and faq update
  • Loading branch information
@sagargg
sagargg authored Apr 25, 2024
2 parents d094136 + 7d60139 commit 52c85aa
Show file tree
Hide file tree
Showing 12 changed files with 996 additions and 943 deletions.
341 changes: 184 additions & 157 deletions ckanext/rvr/actions.py
View file

Large diffs are not rendered by default.

132 changes: 132 additions & 0 deletions ckanext/rvr/assets/css/rvr.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1009,3 +1009,135 @@ div#dataset-map-container.map-draw-field {
margin-top: 383px;
}

/* =======================FAQ PAGE STYLE=========================== */
.faq {
background: inherit;
}

.faq h1 {
font-size: 30px;
font-weight: bold;
margin-top: 20px;
margin-bottom: 20px;
}

.faq-title {
font-size: 22px !important;
text-decoration: underline;
}

.faq-question {
-webkit-text-size-adjust: 100%;
text-align: left;
font-size: 18px !important;
font-weight: bold !important;
position: relative;
padding-left: 30px;
background: inherit;
color: none;
font: inherit;
overflow: visible;
text-transform: none;
font-family: inherit;
margin-bottom: 0;
font-weight: bold;
touch-action: manipulation;
cursor: pointer;
border: none;
background-image: none;
padding: 5px 15px;
line-height: 1.6;
border-radius: 0;
user-select: none;
align-items: baseline;
outline: none;
white-space: normal;
text-align: left;
width: 100%;
display: inline-block;
padding-top: 5px;
padding-bottom: 15px;
left: 10;
}

.faq-question span {
margin-left: 10px;
}

.collapse-indicator {
position: absolute;
top: 25%;
left: 0;
padding-right: -20px;
margin-right: 10px;
font-size: 12px !important; /* adjust as needed */
color: grey; /* adjust as needed */
transition: transform 0.1s !important;
}

.faq-question.collapsed .collapse-indicator {
position: absolute;
padding-right: -20px;
margin-right: 10px;
top: 25% !important;
left: 0 !important;
transform: rotate(180deg);
}

.faq .content {
padding-left: 26px;
}

.collapse {
display: inline-block;
padding-top: 5px;
padding-bottom: 15px;
width: 100%;
}

.collapse p {
font-family: inherit;
font-weight: 300 !important;
display: block;
text-align: left;
font-size: 16px;
}

.collapsed + .collapse {
display: none;
}

@media screen and (max-width: 768px) {
/* Adjust the styles for screens smaller than 768px wide */
.faq {
margin-left: 5px;
margin-right: 5px;
background-color: #eee;
}

.faq-question {
font-size: 20px !important;
margin-left: 5px;
}

.collapse-indicator {
font-size: 2px;
margin-right: 20px;
}

.collapse p {
display: block;
padding-top: 5px;
padding-bottom: 15px;
padding-right: 0px !important;
text-align: left;
font-size: 20px !important;
margin-left: 15px;
margin-right: 15px;
}

.faq-question.collapsed .collapse-indicator {
margin-right: 0px;
}
}
/* =======================END FAQ PAGE STYLE=========================== */
151 changes: 85 additions & 66 deletions ckanext/rvr/commands/migrate_spatial_fields.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@
from ckanext.rvr.helpers import is_valid_spatial, all_package_list

echo = click.echo
context = {
'model': model,
'session': model.Session,
'ignore_auth': True
}
context = {"model": model, "session": model.Session, "ignore_auth": True}

short_help = "Migrates spatial fields for datasets, to prepare them for the spatial inheritance feature of the ckanext-rvr extension"
help = """Migrate spatial fields for datasets and organizations\n
Expand Down Expand Up @@ -49,17 +45,18 @@
migrating them.
"""


@click.command(
name='migrate',
name="migrate",
# There's a bug with ckan 2.9.4 that has been fixed in 2.9.5 where the
# --help flag doesn't work as expected: https://github.com/ckan/ckan/issues/5755
# For now the short_help will display the full help
short_help=help,
help=help,
epilog=epilog,
add_help_option=True
add_help_option=True,
)
@click.option('--dry-run', 'dry_run', is_flag=True, help=dry_run_help)
@click.option("--dry-run", "dry_run", is_flag=True, help=dry_run_help)
def migrate_spatial(dry_run) -> None:
"""
Function handling the `ckan rvr-spatial migrate` command
Expand All @@ -68,23 +65,24 @@ def migrate_spatial(dry_run) -> None:
echo("Getting datasets to migrate...")
mig_dict = get_datasets_to_migrate()

if mig_dict['pkg_count'] < 1:
echo('There are no datasets that need to be migrated.')
if mig_dict["pkg_count"] < 1:
echo("There are no datasets that need to be migrated.")
return

found_message = "Found {} datasets across {} organizations whose spatial fields need to be migrated.".format(
mig_dict['pkg_count'], mig_dict['org_count'])
mig_dict["pkg_count"], mig_dict["org_count"]
)
if dry_run:
message = """DATASETS TO BE MIGRATED BY ORGANIZATION\n\n"""
for org_id in mig_dict['migration_dict']:
title = mig_dict['migration_dict'][org_id]['title']
name = mig_dict['migration_dict'][org_id]['name']
for org_id in mig_dict["migration_dict"]:
title = mig_dict["migration_dict"][org_id]["title"]
name = mig_dict["migration_dict"][org_id]["name"]
message += '<ORGANIZATION title="{}" name={}>\n'.format(title, name)
for data in mig_dict['migration_dict'][org_id]['datasets']:
echo('DATASETS DATA')
for data in mig_dict["migration_dict"][org_id]["datasets"]:
echo("DATASETS DATA")
echo(data)
title = data['title']
name = data['name']
title = data["title"]
name = data["name"]
message += '\t<DATASET title="{}" name={}>\n'.format(title, name)
echo(message)

Expand All @@ -93,22 +91,32 @@ def migrate_spatial(dry_run) -> None:

echo(found_message)
ctx = context.copy()
ctx['allow_partial_update'] = True
ctx['user'] = get_action('get_site_user')(context, {})['name']
for org_id in mig_dict['migration_dict']:
for data in mig_dict['migration_dict'][org_id]['datasets']:
ctx["allow_partial_update"] = True
ctx["user"] = get_action("get_site_user")(context, {})["name"]
for org_id in mig_dict["migration_dict"]:
for data in mig_dict["migration_dict"][org_id]["datasets"]:
try:
echo('\n\nATTEMPTING TO MIGRATE: <DATASET title="{}" name={}>'.format(
data['title'], data['name']))
echo(
'\n\nATTEMPTING TO MIGRATE: <DATASET title="{}" name={}>'.format(
data["title"], data["name"]
)
)
migrate_dataset(data, ctx)
except Exception as e:
echo(e)
echo("UNABLE TO MIGRATE: <DATASET title={} name={}>".format(
data['title'], data['name']))
echo('SUCCESSFULLY MIGRATED <DATASET title="{}" name={}>'.format(
data['title'], data['name']))
echo(
"UNABLE TO MIGRATE: <DATASET title={} name={}>".format(
data["title"], data["name"]
)
)
echo(
'SUCCESSFULLY MIGRATED <DATASET title="{}" name={}>'.format(
data["title"], data["name"]
)
)
return


def get_org_spatials() -> dict:
"""Get all organizations and their spatial data
Expand All @@ -122,32 +130,36 @@ def get_org_spatials() -> dict:
}
"""
# Get all organizations
org_ids = get_action('organization_list')(context, {})
org_ids = get_action("organization_list")(context, {})
org_spatials = {}

# For each organization, get the org spatial from that organization.
for org_id in org_ids:
org_dict = get_action('organization_show')(context, {
"id": org_id,
"include_datasets": False,
"include_users": False,
"include_groups": False,
"include_tags": False,
"include_followers": False
})
org_spatial = ''
for extra in org_dict.get('extras', []):
if extra['key'] == 'org_spatial':
org_spatial = extra['value']
org_spatials[org_dict['id']] = {
'title': org_dict['title'],
'name': org_dict['name'],
'org_spatial': org_spatial
org_dict = get_action("organization_show")(
context,
{
"id": org_id,
"include_datasets": False,
"include_users": False,
"include_groups": False,
"include_tags": False,
"include_followers": False,
},
)
org_spatial = ""
for extra in org_dict.get("extras", []):
if extra["key"] == "org_spatial":
org_spatial = extra["value"]
org_spatials[org_dict["id"]] = {
"title": org_dict["title"],
"name": org_dict["name"],
"org_spatial": org_spatial,
}
del org_dict

return org_spatials


def needs_migration(pkg_dict: dict, org_spatials: dict):
"""Checks if a dataset needs to be migrated
Expand All @@ -165,16 +177,20 @@ def needs_migration(pkg_dict: dict, org_spatials: dict):
(bool | str) : `False` if the dataset doesn't need to be migrated or \
the spatial value if it does
"""
dataset_spatial = pkg_dict['dataset_spatial']
spatial = pkg_dict['spatial']
dataset_spatial = pkg_dict["dataset_spatial"]
spatial = pkg_dict["spatial"]

if is_valid_spatial(dataset_spatial):
return False
elif is_valid_spatial(spatial) and spatial != org_spatials[pkg_dict['owner_org']]['org_spatial']:
elif (
is_valid_spatial(spatial)
and spatial != org_spatials[pkg_dict["owner_org"]]["org_spatial"]
):
return spatial

return False


def get_datasets_to_migrate() -> dict:
"""Gets the datasets that need to be migrated grouped by their organizations
Expand Down Expand Up @@ -202,33 +218,36 @@ def get_datasets_to_migrate() -> dict:
pkg_count = 0
org_count = 0
org_spatials = get_org_spatials()

pkg_ids = all_package_list()
echo('Found a total of {} datasets on the portal. Checking which ones need to be migrated'.format(len(pkg_ids)))
echo(
"Found a total of {} datasets on the portal. Checking which ones need to be migrated".format(
len(pkg_ids)
)
)
for pkg_id in pkg_ids:
data = get_action('package_show')(context, {
"id": pkg_id
})
data = get_action("package_show")(context, {"id": pkg_id})
dataset_spatial = needs_migration(data, org_spatials)
if dataset_spatial:
data['dataset_spatial'] = dataset_spatial
if migration_dict.get(data['owner_org'], None):
migration_dict[data['owner_org']]['datasets'].append(data)
migration_dict[data['owner_org']]['count'] += 1
data["dataset_spatial"] = dataset_spatial
if migration_dict.get(data["owner_org"], None):
migration_dict[data["owner_org"]]["datasets"].append(data)
migration_dict[data["owner_org"]]["count"] += 1
pkg_count += 1
else:
migration_dict[data['owner_org']] = org_spatials[data['owner_org']]
migration_dict[data['owner_org']]['datasets'] = [data]
migration_dict[data['owner_org']]['count'] = 1
migration_dict[data["owner_org"]] = org_spatials[data["owner_org"]]
migration_dict[data["owner_org"]]["datasets"] = [data]
migration_dict[data["owner_org"]]["count"] = 1
org_count += 1
pkg_count += 1
del org_spatials
return {
'migration_dict': migration_dict,
'org_count': org_count,
'pkg_count': pkg_count
"migration_dict": migration_dict,
"org_count": org_count,
"pkg_count": pkg_count,
}


def migrate_dataset(data: dict, context: dict = context):
"""Updates the dataset_spatial field of a dataset
Expand All @@ -237,4 +256,4 @@ def migrate_dataset(data: dict, context: dict = context):
other required fields as per the schema
context (dict, optional): context object. Defaults to script context.
"""
updated = get_action('package_update')(context, data)
updated = get_action("package_update")(context, data)
Loading

0 comments on commit 52c85aa

Please sign in to comment.