Skip to content
This repository has been archived by the owner on Mar 10, 2024. It is now read-only.

Commit

Permalink
feat: expand associations for single GET (salesforce + hubspot only) (#…
Browse files Browse the repository at this point in the history 
…2051)

This is part 1 of expanding associations.

This also includes a large refactor to move `includeRawData` to the
service / provider layers.

This also fixes a bug where we were performing schema mapping twice for
managed data. (The synced data is already mapped, so no need to map it
again)

TODO:
- implement expand for uncached LIST
- implement expand for cached LIST
- apply schemas on expanded objects
  • Loading branch information
@asdfryan
asdfryan authored Dec 12, 2023
1 parent e81f636 commit 92f431a
Show file tree
Hide file tree
Showing 55 changed files with 756 additions and 415 deletions.
4 changes: 2 additions & 2 deletions apps/api/dependency_container.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ const TEMPORAL_ADDRESS =
process.env.SUPAGLUE_TEMPORAL_HOST && process.env.SUPAGLUE_TEMPORAL_PORT
? `${process.env.SUPAGLUE_TEMPORAL_HOST}:${process.env.SUPAGLUE_TEMPORAL_PORT}`
: process.env.SUPAGLUE_TEMPORAL_HOST
? `${process.env.SUPAGLUE_TEMPORAL_HOST}:7233`
: 'temporal';
? `${process.env.SUPAGLUE_TEMPORAL_HOST}:7233`
: 'temporal';

type DependencyContainer = CoreDependencyContainer & {
temporalClient: Client;
Expand Down
68 changes: 27 additions & 41 deletions apps/api/routes/crm/v2/account.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { getDependencyContainer } from '@/dependency_container';
import { NotImplementedError } from '@supaglue/core/errors';
import { BadRequestError, NotImplementedError } from '@supaglue/core/errors';
import { toSnakecasedKeysCrmAccount } from '@supaglue/core/mappers/crm';
import { toMappedProperties } from '@supaglue/core/remotes/utils/properties';
import type {
CreateAccountPathParams,
CreateAccountRequest,
Expand All @@ -21,12 +20,11 @@ import type {
UpsertAccountRequest,
UpsertAccountResponse,
} from '@supaglue/schemas/v2/crm';
import type { FieldMappingConfig } from '@supaglue/types/field_mapping_config';
import { camelcaseKeysSansCustomFields } from '@supaglue/utils/camelcase';
import type { Request, Response } from 'express';
import { Router } from 'express';

const { crmCommonObjectService, managedDataService, connectionService } = getDependencyContainer();
const { crmCommonObjectService, managedDataService } = getDependencyContainer();

export default function init(app: Router): void {
const router = Router();
Expand All @@ -42,46 +40,35 @@ export default function init(app: Router): void {
if (req.query?.read_from_cache?.toString() !== 'true') {
const { pagination, records } = await crmCommonObjectService.list('account', req.customerConnection, {
modifiedAfter: req.query?.modified_after,
expand: req.query?.expand?.split(','),
includeRawData,
cursor: req.query?.cursor,
pageSize: req.query?.page_size ? parseInt(req.query.page_size) : undefined,
associationsToFetch: req.query?.associations_to_fetch,
});
return res.status(200).send({
pagination,
records: records.map((record) => ({
...toSnakecasedKeysCrmAccount(record),
raw_data: includeRawData ? record.rawData : undefined,
})),
records: records.map(toSnakecasedKeysCrmAccount),
});
}
const { pagination, records } = await managedDataService.getCrmAccountRecords(
req.supaglueApplication.id,
req.customerConnection.providerName,
req.customerId,
req.query?.cursor,
req.query?.modified_after as unknown as string | undefined,
req.query?.page_size ? parseInt(req.query.page_size) : undefined
);
let fieldMappingConfig: FieldMappingConfig | undefined = undefined;
if (includeRawData) {
fieldMappingConfig = await connectionService.getFieldMappingConfig(
req.customerConnection.id,
'common',
'account'
);
// TODO: Implement expand for uncached reads
if (req.query?.expand?.length) {
throw new BadRequestError('Expand is not yet supported for uncached reads');
}
return res.status(200).send({
pagination,
records: records.map((record) => ({
...record,
raw_data:
includeRawData && fieldMappingConfig ? toMappedProperties(record.raw_data, fieldMappingConfig) : undefined,
_supaglue_application_id: undefined,
_supaglue_customer_id: undefined,
_supaglue_provider_name: undefined,
_supaglue_emitted_at: undefined,
})),
});
return res
.status(200)
.send(
await managedDataService.getCrmAccountRecords(
req.supaglueApplication.id,
req.customerConnection.providerName,
req.customerConnection.id,
req.customerId,
req.query?.cursor,
req.query?.modified_after as unknown as string | undefined,
req.query?.page_size ? parseInt(req.query.page_size) : undefined,
includeRawData
)
);
}
);

Expand All @@ -91,12 +78,11 @@ export default function init(app: Router): void {
req: Request<GetAccountPathParams, GetAccountResponse, GetAccountRequest, GetAccountQueryParams>,
res: Response<GetAccountResponse>
) => {
const account = await crmCommonObjectService.get(
'account',
req.customerConnection,
req.params.account_id,
req.query?.associations_to_fetch
);
const account = await crmCommonObjectService.get('account', req.customerConnection, req.params.account_id, {
includeRawData: req.query?.include_raw_data?.toString() === 'true',
expand: req.query?.expand?.split(','),
associationsToFetch: req.query?.associations_to_fetch,
});
const snakecasedKeysAccount = toSnakecasedKeysCrmAccount(account);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { raw_data, ...rest } = snakecasedKeysAccount;
Expand Down
64 changes: 27 additions & 37 deletions apps/api/routes/crm/v2/contact.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { getDependencyContainer } from '@/dependency_container';
import { NotImplementedError } from '@supaglue/core/errors';
import { BadRequestError, NotImplementedError } from '@supaglue/core/errors';
import { toSnakecasedKeysCrmContact } from '@supaglue/core/mappers/crm';
import { toMappedProperties } from '@supaglue/core/remotes/utils/properties';
import type {
CreateContactPathParams,
CreateContactRequest,
Expand All @@ -25,12 +24,11 @@ import type {
UpsertContactRequest,
UpsertContactResponse,
} from '@supaglue/schemas/v2/crm';
import type { FieldMappingConfig } from '@supaglue/types/field_mapping_config';
import { camelcaseKeys, camelcaseKeysSansCustomFields } from '@supaglue/utils/camelcase';
import type { Request, Response } from 'express';
import { Router } from 'express';

const { crmCommonObjectService, managedDataService, connectionService } = getDependencyContainer();
const { crmCommonObjectService, managedDataService } = getDependencyContainer();

export default function init(app: Router): void {
const router = Router();
Expand All @@ -47,6 +45,8 @@ export default function init(app: Router): void {
modifiedAfter: req.query?.modified_after,
cursor: req.query?.cursor,
pageSize: req.query?.page_size ? parseInt(req.query.page_size) : undefined,
includeRawData,
expand: req.query?.expand?.split(','),
associationsToFetch: req.query?.associations_to_fetch,
});
return res.status(200).send({
Expand All @@ -57,34 +57,24 @@ export default function init(app: Router): void {
})),
});
}
const { pagination, records } = await managedDataService.getCrmContactRecords(
req.supaglueApplication.id,
req.customerConnection.providerName,
req.customerId,
req.query?.cursor,
req.query?.modified_after as unknown as string | undefined,
req.query?.page_size ? parseInt(req.query.page_size) : undefined
);
let fieldMappingConfig: FieldMappingConfig | undefined = undefined;
if (includeRawData) {
fieldMappingConfig = await connectionService.getFieldMappingConfig(
req.customerConnection.id,
'common',
'contact'
);
// TODO: Implement expand for uncached reads
if (req.query?.expand?.length) {
throw new BadRequestError('Expand is not yet supported for uncached reads');
}
return res.status(200).send({
pagination,
records: records.map((record) => ({
...record,
raw_data:
includeRawData && fieldMappingConfig ? toMappedProperties(record.raw_data, fieldMappingConfig) : undefined,
_supaglue_application_id: undefined,
_supaglue_customer_id: undefined,
_supaglue_provider_name: undefined,
_supaglue_emitted_at: undefined,
})),
});
return res
.status(200)
.send(
await managedDataService.getCrmContactRecords(
req.supaglueApplication.id,
req.customerConnection.providerName,
req.customerConnection.id,
req.customerId,
req.query?.cursor,
req.query?.modified_after as unknown as string | undefined,
req.query?.page_size ? parseInt(req.query.page_size) : undefined,
includeRawData
)
);
}
);

Expand All @@ -94,12 +84,11 @@ export default function init(app: Router): void {
req: Request<GetContactPathParams, GetContactResponse, GetContactRequest, GetContactQueryParams>,
res: Response<GetContactResponse>
) => {
const contact = await crmCommonObjectService.get(
'contact',
req.customerConnection,
req.params.contact_id,
req.query?.associations_to_fetch
);
const contact = await crmCommonObjectService.get('contact', req.customerConnection, req.params.contact_id, {
includeRawData: req.query?.include_raw_data?.toString() === 'true',
expand: req.query?.expand?.split(','),
associationsToFetch: req.query?.associations_to_fetch,
});
const snakecasedKeysContact = toSnakecasedKeysCrmContact(contact);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { raw_data, ...rest } = snakecasedKeysContact;
Expand Down Expand Up @@ -143,6 +132,7 @@ export default function init(app: Router): void {
res: Response<SearchContactsResponse>
) => {
const { pagination, records } = await crmCommonObjectService.search('contact', req.customerConnection, {
includeRawData: req.query?.include_raw_data?.toString() === 'true',
filter: req.body.filter,
cursor: req.query?.cursor,
pageSize: req.query?.page_size ? parseInt(req.query.page_size) : undefined,
Expand Down
71 changes: 29 additions & 42 deletions apps/api/routes/crm/v2/lead.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { getDependencyContainer } from '@/dependency_container';
import { NotImplementedError } from '@supaglue/core/errors';
import { BadRequestError, NotImplementedError } from '@supaglue/core/errors';
import { toSnakecasedKeysCrmLead } from '@supaglue/core/mappers/crm';
import { toMappedProperties } from '@supaglue/core/remotes/utils/properties';
import type {
CreateLeadPathParams,
CreateLeadRequest,
Expand All @@ -25,12 +24,11 @@ import type {
UpsertLeadRequest,
UpsertLeadResponse,
} from '@supaglue/schemas/v2/crm';
import type { FieldMappingConfig } from '@supaglue/types/field_mapping_config';
import { camelcaseKeys, camelcaseKeysSansCustomFields } from '@supaglue/utils/camelcase';
import type { Request, Response } from 'express';
import { Router } from 'express';

const { crmCommonObjectService, managedDataService, connectionService } = getDependencyContainer();
const { crmCommonObjectService, managedDataService } = getDependencyContainer();

export default function init(app: Router): void {
const router = Router();
Expand All @@ -41,11 +39,11 @@ export default function init(app: Router): void {
req: Request<GetLeadPathParams, GetLeadResponse, GetLeadRequest, GetLeadQueryParams>,
res: Response<GetLeadResponse>
) => {
const lead = await crmCommonObjectService.get('lead', req.customerConnection, req.params.lead_id);
const snakecasedKeysLead = toSnakecasedKeysCrmLead(lead);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { raw_data, ...rest } = snakecasedKeysLead;
return res.status(200).send(req.query?.include_raw_data?.toString() === 'true' ? snakecasedKeysLead : rest);
const lead = await crmCommonObjectService.get('lead', req.customerConnection, req.params.lead_id, {
includeRawData: req.query?.include_raw_data?.toString() === 'true',
expand: req.query?.expand?.split(','),
});
return res.status(200).send(toSnakecasedKeysCrmLead(lead));
}
);

Expand All @@ -60,41 +58,34 @@ export default function init(app: Router): void {
if (req.query?.read_from_cache?.toString() !== 'true') {
const { pagination, records } = await crmCommonObjectService.list('lead', req.customerConnection, {
modifiedAfter: req.query?.modified_after,
includeRawData,
expand: req.query?.expand?.split(','),
cursor: req.query?.cursor,
pageSize: req.query?.page_size ? parseInt(req.query.page_size) : undefined,
});
return res.status(200).send({
pagination,
records: records.map((record) => ({
...toSnakecasedKeysCrmLead(record),
raw_data: includeRawData ? record.rawData : undefined,
})),
records: records.map(toSnakecasedKeysCrmLead),
});
}
const { pagination, records } = await managedDataService.getCrmLeadRecords(
req.supaglueApplication.id,
req.customerConnection.providerName,
req.customerId,
req.query?.cursor,
req.query?.modified_after as unknown as string | undefined,
req.query?.page_size ? parseInt(req.query.page_size) : undefined
);
let fieldMappingConfig: FieldMappingConfig | undefined = undefined;
if (includeRawData) {
fieldMappingConfig = await connectionService.getFieldMappingConfig(req.customerConnection.id, 'common', 'lead');
// TODO: Implement expand for uncached reads
if (req.query?.expand?.length) {
throw new BadRequestError('Expand is not yet supported for uncached reads');
}
return res.status(200).send({
pagination,
records: records.map((record) => ({
...record,
raw_data:
includeRawData && fieldMappingConfig ? toMappedProperties(record.raw_data, fieldMappingConfig) : undefined,
_supaglue_application_id: undefined,
_supaglue_customer_id: undefined,
_supaglue_provider_name: undefined,
_supaglue_emitted_at: undefined,
})),
});

return res
.status(200)
.send(
await managedDataService.getCrmLeadRecords(
req.supaglueApplication.id,
req.customerConnection.providerName,
req.customerId,
req.customerConnection.id,
req.query?.cursor,
req.query?.modified_after as unknown as string | undefined,
req.query?.page_size ? parseInt(req.query.page_size) : undefined
)
);
}
);

Expand Down Expand Up @@ -151,15 +142,11 @@ export default function init(app: Router): void {
filter: req.body.filter,
cursor: req.query?.cursor,
pageSize: req.query?.page_size ? parseInt(req.query.page_size) : undefined,
includeRawData: req.query?.include_raw_data?.toString() === 'true',
});
return res.status(200).send({
pagination,
records: records.map((record) => {
const snakecased = toSnakecasedKeysCrmLead(record);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { raw_data, ...rest } = snakecased;
return req.query?.include_raw_data?.toString() === 'true' ? snakecased : rest;
}),
records: records.map(toSnakecasedKeysCrmLead),
});
}
);
Expand Down
Loading

1 comment on commit 92f431a

@vercel
Copy link

@vercel vercel bot commented on 92f431a Dec 12, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

supaglue-docs – ./docs

supaglue-docs-git-main-supaglue.vercel.app
supaglue-docs-supaglue.vercel.app
docs.supaglue.com

Please sign in to comment.