Merge pull request #1337 from research-software-directory/1333-expand…

…-ror-scraper

Save ROR data to RSD and use it

database/013-create-organisation-table.sql

@@ -16,11 +16,15 @@ CREATE TABLE organisation (
 	name VARCHAR(200) NOT NULL,
 	short_description VARCHAR(300),
 	description VARCHAR(10000),
-	ror_id VARCHAR(100) UNIQUE,
+	ror_id VARCHAR(100) UNIQUE CHECK (ror_id ~ '^https://ror\.org/(0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2})$'),
 	website VARCHAR(200) UNIQUE,
 	is_tenant BOOLEAN DEFAULT FALSE NOT NULL,
 	country VARCHAR(100),
 	city VARCHAR(100),
+	wikipedia_url VARCHAR(300),
+	ror_types VARCHAR(100)[],
+	lat float8,
+	lon float8,
 	ror_scraped_at TIMESTAMPTZ,
 	ror_last_error VARCHAR(500),
 	logo_id VARCHAR(40) REFERENCES image(id),

frontend/components/admin/organisations/apiOrganisation.tsx

@@ -13,7 +13,7 @@ import {paginationUrlParams} from '~/utils/postgrestUrl'
 import {createJsonHeaders, getBaseUrl} from '~/utils/fetchHelpers'
 import {extractCountFromHeader} from '~/utils/extractCountFromHeader'
 import logger from '~/utils/logger'
-import {columsForCreate, EditOrganisation, OrganisationList} from '~/types/Organisation'
+import {colForCreate, EditOrganisation, OrganisationList} from '~/types/Organisation'
 import {upsertImage} from '~/utils/editImage'
 import {getSlugFromString} from '~/utils/getSlugFromString'
 import {getPropsFromObject} from '~/utils/getPropsFromObject'
@@ -137,7 +137,7 @@ export function useOrganisations(token: string) {
       // create slug for new organisation based on name
       data.slug = getSlugFromString(data.name)
       // extract props we need for createOrganisation
-      const organisation = getPropsFromObject(data, columsForCreate)
+      const organisation = getPropsFromObject(data, colForCreate)
       // create new organisation
       const {status,message} = await createOrganisation({
         organisation,

frontend/components/organisation/apiOrganisations.ts

@@ -9,9 +9,8 @@
 import {RsdUser} from '~/auth'
 import {isOrganisationMaintainer} from '~/auth/permissions/isMaintainerOfOrganisation'
 import {
-  Organisation, OrganisationForOverview,
-  OrganisationList, ProjectOfOrganisation,
-  SoftwareOfOrganisation
+  OrganisationForOverview, OrganisationList,
+  ProjectOfOrganisation, SoftwareOfOrganisation
 } from '~/types/Organisation'
 import {extractCountFromHeader} from '~/utils/extractCountFromHeader'
 import {createJsonHeaders, getBaseUrl} from '~/utils/fetchHelpers'
@@ -92,19 +91,19 @@ export async function getOrganisationBySlug({slug,user,token}:
     })
     // console.log('getOrganisationBySlug...isMaintainer...', isMaintainer)
     // get organisation data
-    const [organisation, description] = await Promise.all([
+    const [organisation, orgInfo] = await Promise.all([
       getOrganisationById({
         uuid,
         token,
         isMaintainer
       }),
-      getOrganisationDescription({uuid, token})
+      getOrganisationInfo({uuid, token})
     ])
-    // return consolidate organisation
+    // return consolidated organisation data
     return {
       organisation: {
         ...organisation,
-        description
+        ...orgInfo
       },
       isMaintainer
     }
@@ -193,8 +192,30 @@ export async function getOrganisationChildren({uuid, token}:
   return []
 }
 
-export async function getOrganisationDescription({uuid, token}: { uuid: string, token?: string }) {
-  const query = `organisation?id=eq.${uuid}&select=description`
+// export async function getOrganisationDescription({uuid, token}: { uuid: string, token?: string }) {
+//   const query = `organisation?id=eq.${uuid}&select=description`
+//   const url = `${getBaseUrl()}/${query}`
+//   // console.log('url...', url)
+//   const resp = await fetch(url, {
+//     method: 'GET',
+//     headers: {
+//       ...createJsonHeaders(token),
+//       // request single object item
+//       'Accept': 'application/vnd.pgrst.object+json'
+//     }
+//   })
+//   if (resp.status === 200) {
+//     const json: Organisation = await resp.json()
+//     return json.description
+//   }
+//   // otherwise request failed
+//   logger(`getOrganisationDescription failed: ${resp.status} ${resp.statusText}`, 'warn')
+//   // we log and return null
+//   return null
+// }
+
+export async function getOrganisationInfo({uuid, token}: { uuid: string, token?: string }) {
+  const query = `organisation?id=eq.${uuid}&select=description,wikipedia_url,city,ror_types`
   const url = `${getBaseUrl()}/${query}`
   // console.log('url...', url)
   const resp = await fetch(url, {
@@ -206,8 +227,13 @@ export async function getOrganisationDescription({uuid, token}: { uuid: string,
     }
   })
   if (resp.status === 200) {
-    const json: Organisation = await resp.json()
-    return json.description
+    const json: any = await resp.json()
+    return {
+      city: json.city as string | null,
+      description: json.description as string | null,
+      wikipedia_url: json.wikipedia_url as string | null,
+      ror_types: json?.ror_types ?? [] as string[] | null
+    }
   }
   // otherwise request failed
   logger(`getOrganisationDescription failed: ${resp.status} ${resp.statusText}`, 'warn')

frontend/components/organisation/context/OrganisationContext.tsx

@@ -11,8 +11,15 @@ type UpdateOrganisationProps = {
   value: any
 }
 
+export type OrganisationForContext = OrganisationForOverview & {
+  description: string | null,
+  wikipedia_url: string | null,
+  city: string | null
+  ror_types: string[] | null
+}
+
 type OrganisationContextProps = {
-  organisation: OrganisationForOverview | null
+  organisation: OrganisationForContext | null
   isMaintainer: boolean
   updateOrganisation:({key,value}:UpdateOrganisationProps)=>void
 }
@@ -24,10 +31,10 @@ const OrganisationContext = createContext<OrganisationContextProps>({
 })
 
 export function OrganisationProvider(props: any) {
-  // destucture organisation
+  // destructure organisation
   const {organisation:initOrganisation, isMaintainer:initMaintainer} = props
   // set state - use initOrganisation at start
-  const [organisation, setOrganisation] = useState<OrganisationForOverview | null>(initOrganisation)
+  const [organisation, setOrganisation] = useState<OrganisationForContext | null>(initOrganisation)
   const [isMaintainer, setIsMaintainer] = useState<boolean>(initMaintainer ?? false)
 
   const updateOrganisation = useCallback(({key, value}:UpdateOrganisationProps) => {
@@ -40,7 +47,7 @@ export function OrganisationProvider(props: any) {
     }
   },[organisation])
 
-  // we need to update organisation state every time initOrganistation changes
+  // we need to update organisation state every time initOrganisation changes
   // because useState is running in different context
   useEffect(() => {
     if (initOrganisation.id && !organisation) {

frontend/components/organisation/metadata/RorType.tsx

@@ -5,16 +5,15 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-import {RORItem} from '~/utils/getROR'
 import TypeIcon from '~/components/icons/TypeIcon'
 
-export default function RorType({meta}:{meta:RORItem|null}) {
+export default function RorType({ror_types}:Readonly<{ror_types:string[]|null}>) {
   try {
-    if (meta === null) return null
+    if (ror_types === null) return null
 
     return (
       <>
-        {meta.types.map(item => (
+        {ror_types.map(item => (
           <div key={item} className="flex gap-2">
             <TypeIcon />
             <span>{item}</span>

frontend/components/organisation/metadata/index.tsx

@@ -1,15 +1,14 @@
 // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all)
 // SPDX-FileCopyrightText: 2022 dv4all
-// SPDX-FileCopyrightText: 2023 Dusan Mijatovic (Netherlands eScience Center)
-// SPDX-FileCopyrightText: 2023 Netherlands eScience Center
+// SPDX-FileCopyrightText: 2023 - 2024 Dusan Mijatovic (Netherlands eScience Center)
+// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center
 //
 // SPDX-License-Identifier: Apache-2.0
 
 import LanguageIcon from '@mui/icons-material/Language'
 import AutoStoriesIcon from '@mui/icons-material/AutoStories'
 import MapIcon from '@mui/icons-material/Map'
 
-import {RORItem} from '~/utils/getROR'
 import RorIcon from '~/components/icons/RorIcon'
 import OrganisationLogo from './OrganisationLogo'
 import RorType from './RorType'
@@ -19,12 +18,11 @@ import {getHostnameFromUrl} from '~/utils/getHostname'
 import BaseSurfaceRounded from '~/components/layout/BaseSurfaceRounded'
 import useOrganisationContext from '../context/useOrganisationContext'
 
-type OrganisationMetadataProps = {
-  ror_info: RORItem | null
-}
-
-export default function OrganisationMetadata({ror_info}: OrganisationMetadataProps) {
-  const {name,short_description,country,website,isMaintainer} = useOrganisationContext()
+export default function OrganisationMetadata() {
+  const {
+    name,short_description,country,city,website,
+    isMaintainer,wikipedia_url,ror_id,ror_types
+  } = useOrganisationContext()
 
   // console.group('OrganisationMetadata')
   // console.log('short_description...', short_description)
@@ -45,34 +43,27 @@ export default function OrganisationMetadata({ror_info}: OrganisationMetadataPro
           icon: <LanguageIcon />,
         })
       }
-    } else if (ror_info && ror_info.links && ror_info.links.length > 0) {
-      const title = getHostnameFromUrl(ror_info.links[0]) ?? 'Website'
-      rsdLinks.push({
-        title,
-        url: ror_info.links[0],
-        icon: <LanguageIcon />,
-      })
     }
-    // ror_info.id is ror_id url
-    if (ror_info && ror_info.id) {
+    // ror_id url
+    if (ror_id) {
       // add only new items
       rsdLinks.push({
         title:'ROR info',
-        url: ror_info.id,
+        url: ror_id,
         icon: <RorIcon/>,
       })
     }
     // some organisations provide wikipedia page
-    if (ror_info && ror_info?.wikipedia_url) {
+    if (wikipedia_url) {
       rsdLinks.push({
         title:'Wikipedia',
-        url: ror_info?.wikipedia_url,
+        url: wikipedia_url,
         icon: <AutoStoriesIcon />,
       })
     }
     // Google Maps link
-    if (ror_info?.addresses[0].city && ror_info?.country.country_name) {
-      const query = encodeURIComponent(`${name},${ror_info?.addresses[0].city},${ror_info?.country.country_name}`)
+    if (name && city && country) {
+      const query = encodeURIComponent(`${name},${city},${country}`)
       const href = `https://www.google.com/maps/search/?api=1&query=${query}`
       rsdLinks.push({
         title:'Map',
@@ -97,15 +88,15 @@ export default function OrganisationMetadata({ror_info}: OrganisationMetadataPro
             {name}
           </h1>
           <RorLocation
-            city={ror_info?.addresses[0].city ?? null}
-            country={ror_info?.country.country_name ?? country ?? null}
+            city={city ?? null}
+            country={country ?? null}
           />
           <p className="text-base-700 line-clamp-3 break-words py-4">
             {short_description}
           </p>
         </div>
         <div className="flex flex-col gap-4">
-          <RorType meta={ror_info} />
+          <RorType ror_types={ror_types ?? null} />
           <Links links={getAllLinks()} />
         </div>
       </BaseSurfaceRounded>

frontend/components/organisation/units/OrganisationUnitsIndex.test.tsx

@@ -264,7 +264,7 @@ describe('frontend/components/organisation/software/index.tsx', () => {
     // validate api call
     const expectedUrl = '/api/v1/organisation'
     const expectedPayload = {
-      'body': '{"parent":"91c2ffa7-bce6-4488-be00-6613a2d99f51","slug":"test-unit-name","primary_maintainer":"121212121212","name":"Test unit name","ror_id":null,"is_tenant":false,"website":"https://google.com/test","logo_id":null}',
+      'body': '{"parent":"91c2ffa7-bce6-4488-be00-6613a2d99f51","primary_maintainer":"121212121212","slug":"test-unit-name","name":"Test unit name","ror_id":null,"website":"https://google.com/test","is_tenant":false,"country":null,"city":null,"wikipedia_url":null,"ror_types":null,"logo_id":null}',
       'headers': {
         'Authorization': 'Bearer TEST_TOKEN',
         'Content-Type': 'application/json',

frontend/components/organisation/units/index.tsx

@@ -15,7 +15,7 @@ import Button from '@mui/material/Button'
 import {useSession} from '~/auth'
 import useSnackbar from '../../snackbar/useSnackbar'
 import {
-  columsForCreate, columsForUpdate, CoreOrganisationProps,
+  colForCreate, colForUpdate, CoreOrganisationProps,
   EditOrganisation, Organisation, OrganisationForOverview
 } from '../../../types/Organisation'
 import {
@@ -150,7 +150,7 @@ export default function ResearchUnits() {
       }
       // SAVE organisation
       if (typeof pos != 'undefined' && data.id) {
-        const unit:Organisation = getPropsFromObject(data,columsForUpdate)
+        const unit:Organisation = getPropsFromObject(data,colForUpdate)
         // update existing organisation
         const resp = await updateOrganisation({
           organisation: unit,
@@ -165,7 +165,7 @@ export default function ResearchUnits() {
         }
       } else {
         // create new organisation
-        const unit:CoreOrganisationProps = getPropsFromObject(data, columsForCreate)
+        const unit:CoreOrganisationProps = getPropsFromObject(data, colForCreate)
         const resp = await createOrganisation({
           organisation:unit,
           token

frontend/components/projects/edit/information/AutosaveFundingOrganisations.test.tsx

@@ -182,14 +182,18 @@ it('can add funding organisation from ROR', async() => {
     expect(mockCreateOrganisation).toBeCalledTimes(1)
     expect(mockCreateOrganisation).toBeCalledWith({
       'organisation': {
+        'city': null,
+        'country': null,
         'is_tenant': false,
         'logo_id': foundOrgs[1].data.logo_id,
         'name': foundOrgs[1].data.name,
         'parent': null,
         'primary_maintainer': null,
         'ror_id': foundOrgs[1].data.ror_id,
+        'ror_types': null,
         'slug': 'vu-university-amsterdam',
-        'website': foundOrgs[1].data.website
+        'website': foundOrgs[1].data.website,
+        'wikipedia_url': null,
       },
       'token': 'TEST_TOKEN',
     })

frontend/components/projects/edit/information/AutosaveFundingOrganisations.tsx

@@ -11,7 +11,7 @@ import {useState} from 'react'
 
 import {useSession} from '~/auth'
 import useSnackbar from '~/components/snackbar/useSnackbar'
-import {columsForCreate, SearchOrganisation} from '~/types/Organisation'
+import {colForCreate, SearchOrganisation} from '~/types/Organisation'
 import {createOrganisation, searchForOrganisation} from '~/utils/editOrganisation'
 import {addOrganisationToProject, deleteOrganisationFromProject} from '~/utils/editProject'
 import {getPropsFromObject} from '~/utils/getPropsFromObject'
@@ -40,7 +40,7 @@ export default function AutosaveFundingOrganisations({id,items}:FundingOrganisat
       }
       // console.log('onAddOrganisation...', selected)
       if (selected.id===null){
-        const organisation = getPropsFromObject(selected,columsForCreate)
+        const organisation = getPropsFromObject(selected,colForCreate)
         // createNewOrganisation(selected)
         resp = await createOrganisation({
           organisation,

frontend/components/projects/edit/organisations/EditProjectOrganisationsIndex.test.tsx

@@ -198,14 +198,18 @@ describe('frontend/components/projects/edit/organisations/index.tsx', () => {
       expect(mockCreateOrganisation).toBeCalledTimes(1)
       expect(mockCreateOrganisation).toBeCalledWith({
         'organisation': {
+          'city': null,
+          'country': null,
           'is_tenant': false,
           'logo_id': null,
           'name': searchFor,
           'parent': null,
           'primary_maintainer': null,
           'ror_id': null,
+          'ror_types': null,
           'slug': expectSlug,
           'website': expectWebsite,
+          'wikipedia_url': null,
         },
         'token': mockSession.token,
       })

frontend/components/projects/edit/organisations/index.tsx

@@ -9,7 +9,7 @@
 import {useState} from 'react'
 
 import {useSession} from '~/auth'
-import {columsForUpdate, EditOrganisation, SearchOrganisation} from '~/types/Organisation'
+import {colForUpdate, EditOrganisation, SearchOrganisation} from '~/types/Organisation'
 import {
   newOrganisationProps,
   searchToEditOrganisation, updateOrganisation
@@ -230,7 +230,7 @@ export default function ProjectOrganisations() {
       // SAVE organisation
       if (typeof pos != 'undefined' && data.id) {
         // extract data for update
-        const organisation = getPropsFromObject(data,columsForUpdate)
+        const organisation = getPropsFromObject(data,colForUpdate)
         // update existing organisation
         const resp = await updateOrganisation({
           organisation,