From 1f743da6007d32f167437710bbdf039b9c82fda5 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Mon, 16 Sep 2024 18:49:35 -0400 Subject: [PATCH 01/11] change resolver code based on new gql type --- .../src/network/resolvers/aggregations.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index 15452999c..58445c03d 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -141,6 +141,25 @@ const createGqlFieldsString = (config: NetworkAggregationConfig, requestedAggreg }, ''); }; +/** + * Parse central query and build individual queries for remote connections based on available fields + * + * @param requestedAggregations + */ +const createGqlFieldsString = (config: NetworkAggregationConfig, requestedAggregations: any[]) => { + return requestedAggregations.reduce((gqlString, fieldName) => { + const matchedAggregationField = findMatchedAggregationField(config, fieldName); + if (matchedAggregationField) { + const { name, type } = matchedAggregationField; + // get gql query string for supported aggregation + // TODO: only query requested fields + bucket_count if nodes is requested + const aggregationFieldQueryString = supportedAggregationQueries.get(type); + return gqlString + `${name}${aggregationFieldQueryString}`; + } + return gqlString; + }, ''); +}; + /** * Create queries for remote nodes based on requested fields * From a088522c198b3222ca9f117d506489d294edfeef Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Wed, 18 Sep 2024 17:54:06 -0400 Subject: [PATCH 02/11] comments and cleanup --- modules/server/src/network/resolvers/aggregations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index 58445c03d..ef74c3bde 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -142,7 +142,7 @@ const createGqlFieldsString = (config: NetworkAggregationConfig, requestedAggreg }; /** - * Parse central query and build individual queries for remote connections based on available fields + * Create queries for remote nodes based on requested fields * * @param requestedAggregations */ From e7bc1c909f0d335ff1a037e5723d871e6b7b7163 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Thu, 19 Sep 2024 00:43:24 -0400 Subject: [PATCH 03/11] break up main entry point into two modules, one for querying and one for field processing --- modules/server/src/network/index.ts | 145 ++------------------- modules/server/src/network/setup/fields.ts | 68 ++++++++++ modules/server/src/network/setup/query.ts | 75 +++++++++++ modules/server/src/network/util.ts | 9 -- 4 files changed, 151 insertions(+), 146 deletions(-) create mode 100644 modules/server/src/network/setup/fields.ts create mode 100644 modules/server/src/network/setup/query.ts diff --git a/modules/server/src/network/index.ts b/modules/server/src/network/index.ts index c7c59f06e..c604ebd74 100644 --- a/modules/server/src/network/index.ts +++ b/modules/server/src/network/index.ts @@ -1,144 +1,15 @@ import { makeExecutableSchema } from '@graphql-tools/schema'; -import { SUPPORTED_AGGREGATIONS_LIST } from './common'; -import { NetworkAggregationError } from './errors'; -import { fetchGql } from './gql'; -import { gqlAggregationTypeQuery, GQLFieldType, GQLTypeQueryResponse } from './queries'; +import { SUPPORTED_AGGREGATIONS, SUPPORTED_AGGREGATIONS_LIST } from './common'; import { createResolvers } from './resolvers'; +import { getAllFieldTypes } from './setup/fields'; +import { fetchRemoteSchemas } from './setup/query'; import { createTypeDefs } from './typeDefs'; -import { - NetworkAggregationConfig, - NetworkAggregationConfigInput, - NetworkFieldType, - SupportedNetworkFieldType, -} from './types'; -import { fulfilledPromiseFilter, getAllTypes } from './util'; - -/** - * GQL query remote connection with __type query to retrieve list of types - * - * @param config - network config from env - * @returns a promise containing network config and the gql query result - * - * @throws Fetch failed error - * - * @throws JSON parse error - * - * @throws Unexpected data error - */ -const fetchRemoteSchema = async ( - config: NetworkAggregationConfigInput, -): Promise => { - const { graphqlUrl, documentType, timeout } = config; - try { - const response = await fetchGql({ - url: graphqlUrl, - gqlQuery: gqlAggregationTypeQuery, - variables: { documentName: documentType }, - options: { - timeout, - }, - }); - - // axios response "data" field, graphql response "data" field - const responseData = response.data?.data; - if (response.status === 200 && response.statusText === 'OK') { - return responseData; - } - - console.error('unexpected response data in fetchRemoteSchema'); - throw new NetworkAggregationError( - `Unexpected data in response object. Please verify the endpoint at ${graphqlUrl} is returning a valid GQL Schema.`, - ); - } catch (error) { - console.error(`Failed to retrieve schema from url: ${config.graphqlUrl}`); - return; - } -}; - -const isSupportedType = ( - fieldObject: NetworkFieldType, - supportedList: string[], -): fieldObject is SupportedNetworkFieldType => { - return supportedList.includes(fieldObject.type); -}; - -/** - * Reduce response into simplified object, returning both supported and unsupported types - * @param fields - * @returns { supportedAggregations: [], unsupportedAggregations: [] } - */ -export const getFieldTypes = (fields: GQLFieldType[], supportedAggregationsList: string[]) => { - const fieldTypes = fields.reduce( - ( - aggregations: Pick< - NetworkAggregationConfig, - 'supportedAggregations' | 'unsupportedAggregations' - >, - field, - ) => { - const fieldType = field.type.name; - const fieldObject = { name: field.name, type: fieldType }; - if (isSupportedType(fieldObject, supportedAggregationsList)) { - return { - ...aggregations, - supportedAggregations: aggregations.supportedAggregations.concat(fieldObject), - }; - } else { - return { - ...aggregations, - unsupportedAggregations: aggregations.unsupportedAggregations.concat(fieldObject), - }; - } - }, - { - supportedAggregations: [], - unsupportedAggregations: [], - }, - ); - - return fieldTypes; -}; - -type NetworkQueryResult = PromiseFulfilledResult<{ - config: NetworkAggregationConfigInput; - gqlResponse: GQLTypeQueryResponse; -}>; - -/** - * Fetch fields and types from remote connections - * @param { networkConfigs } - * @returns GQL object type to be used in functions - */ -const fetchRemoteSchemas = async ({ - networkConfigs, -}: { - networkConfigs: NetworkAggregationConfigInput[]; -}): Promise => { - // query remote connection types - const networkQueryPromises = networkConfigs.map(async (config) => { - const gqlResponse = await fetchRemoteSchema(config); - return { config, gqlResponse }; - }); - - const networkQueries = await Promise.allSettled(networkQueryPromises); - const configs = networkQueries - .filter(fulfilledPromiseFilter) - .map((networkResult) => { - const { config, gqlResponse } = networkResult.value; - const fields = gqlResponse.__type.fields; - - const { supportedAggregations, unsupportedAggregations } = getFieldTypes( - fields, - SUPPORTED_AGGREGATIONS_LIST, - ); - return { ...config, supportedAggregations, unsupportedAggregations }; - }); - - return configs; -}; +import { NetworkAggregationConfigInput } from './types'; /** + * GQL Federated Search schema setup * Connects to remote network connections, looks up field types, add field/type pairs to configs + * * @param { networkConfigs } * @returns graphql schema for the network - types and resolvers combined */ @@ -147,11 +18,11 @@ export const createSchemaFromNetworkConfig = async ({ }: { networkConfigs: NetworkAggregationConfigInput[]; }) => { - const configs = await fetchRemoteSchemas({ + const networkFields = await fetchRemoteSchemas({ networkConfigs, }); - const networkFieldTypes = getAllTypes(configs); + const networkFieldTypes = getAllFieldTypes(networkFields, SUPPORTED_AGGREGATIONS_LIST); const typeDefs = createTypeDefs(networkFieldTypes); const resolvers = createResolvers(configs); diff --git a/modules/server/src/network/setup/fields.ts b/modules/server/src/network/setup/fields.ts new file mode 100644 index 000000000..bb68d7ea6 --- /dev/null +++ b/modules/server/src/network/setup/fields.ts @@ -0,0 +1,68 @@ +import { SupportedAggregation } from '../common'; +import { GQLFieldType } from '../queries'; +import { NetworkAggregationConfig, NetworkFieldType, SupportedNetworkFieldType } from '../types'; + +export type NetworkFields = { name: string; fields: GQLFieldType[] }; + +const isSupportedType = ( + fieldObject: NetworkFieldType, + supportedList: string[], +): fieldObject is SupportedNetworkFieldType => { + return supportedList.includes(fieldObject.type); +}; + +/** + * Reduce response into simplified object, returning both supported and unsupported types + * @param fields + * @returns { supportedAggregations: [], unsupportedAggregations: [] } + */ +export const getFieldTypes = ( + fields: GQLFieldType[], + supportedAggregationsList: SupportedAggregation[], +) => { + const fieldTypes = fields.reduce( + ( + aggregations: Pick< + NetworkAggregationConfig, + 'supportedAggregations' | 'unsupportedAggregations' + >, + field, + ) => { + const fieldType = field.type.name; + const fieldObject = { name: field.name, type: fieldType }; + if (isSupportedType(fieldObject, supportedAggregationsList)) { + return { + ...aggregations, + supportedAggregations: aggregations.supportedAggregations.concat(fieldObject), + }; + } else { + return { + ...aggregations, + unsupportedAggregations: aggregations.unsupportedAggregations.concat(fieldObject), + }; + } + }, + { + supportedAggregations: [], + unsupportedAggregations: [], + }, + ); + + return fieldTypes; +}; + +export const getAllFieldTypes = ( + networkFields: NetworkFields[], + supportedTypes: SupportedAggregation[], +) => { + const nodeFieldTypes = networkFields.map((networkField) => { + const { supportedAggregations, unsupportedAggregations } = getFieldTypes( + networkField.fields, + supportedTypes, + ); + + return { supportedAggregations, unsupportedAggregations }; + }); + + return nodeFieldTypes.flatMap((fieldType) => fieldType.supportedAggregations); +}; diff --git a/modules/server/src/network/setup/query.ts b/modules/server/src/network/setup/query.ts new file mode 100644 index 000000000..7468e5771 --- /dev/null +++ b/modules/server/src/network/setup/query.ts @@ -0,0 +1,75 @@ +import { NetworkAggregationError } from '../errors'; +import { fetchGql } from '../gql'; +import { gqlAggregationTypeQuery, GQLTypeQueryResponse } from '../queries'; +import { NetworkAggregationConfigInput } from '../types'; +import { fulfilledPromiseFilter } from '../util'; +import { NetworkFields } from './fields'; + +type NetworkQueryResult = PromiseFulfilledResult<{ + config: NetworkAggregationConfigInput; + gqlResponse: GQLTypeQueryResponse; +}>; + +/** + * GQL query remote connection with __type query to retrieve list of types + * + * @param config - network config from env + * @returns a promise containing network config and the gql query result + * + * @throws Fetch failed error + * + * @throws JSON parse error + * + * @throws Unexpected data error + */ +const fetchRemoteSchema = async ( + config: NetworkAggregationConfigInput, +): Promise => { + const { graphqlUrl, documentType } = config; + try { + const response = await fetchGql({ + url: graphqlUrl, + gqlQuery: gqlAggregationTypeQuery, + variables: { documentName: documentType }, + }); + + // axios response "data" field, graphql response "data" field + const responseData = response.data?.data; + if (response.status === 200 && response.statusText === 'OK') { + return responseData; + } + + console.error('unexpected response data in fetchRemoteSchema'); + throw new NetworkAggregationError( + `Unexpected data in response object. Please verify the endpoint at ${graphqlUrl} is returning a valid GQL Schema.`, + ); + } catch (error) { + console.error(`Failed to retrieve schema from url: ${config.graphqlUrl}`); + return; + } +}; + +/** + * Fetch fields and types from remote connections + * @param { networkConfigs } + * @returns GQL object type to be used in functions + */ +export const fetchRemoteSchemas = async ({ + networkConfigs, +}: { + networkConfigs: NetworkAggregationConfigInput[]; +}): Promise => { + // query remote connection types + const networkQueryPromises = networkConfigs.map(async (config) => { + const gqlResponse = await fetchRemoteSchema(config); + return { config, gqlResponse }; + }); + + const networkQueries = await Promise.allSettled(networkQueryPromises); + + return networkQueries.filter(fulfilledPromiseFilter).map((networkResult) => { + const { config, gqlResponse } = networkResult.value; + const fields = gqlResponse.__type.fields; + return { name: config.displayName, fields }; + }); +}; diff --git a/modules/server/src/network/util.ts b/modules/server/src/network/util.ts index def87618d..d279a013e 100644 --- a/modules/server/src/network/util.ts +++ b/modules/server/src/network/util.ts @@ -2,15 +2,6 @@ import { DocumentNode, GraphQLResolveInfo, print } from 'graphql'; import graphqlFields from 'graphql-fields'; import { NetworkAggregationConfig, SupportedNetworkFieldType } from './types'; -/** - * Returns an array of all field/type - * @param configs - * @returns Returns an array of all field/type - */ -export const getAllTypes = (configs: NetworkAggregationConfig[]): SupportedNetworkFieldType[] => { - return configs.flatMap((config) => config.supportedAggregations); -}; - /* * GraphQL AST => String */ From b4c7230bf197e48090c47a284deb524a26d76976 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Thu, 19 Sep 2024 01:05:25 -0400 Subject: [PATCH 04/11] rename network config type, remove supported aggs from type --- modules/server/src/network/index.ts | 7 ++++--- modules/server/src/network/resolvers/aggregations.ts | 10 +++++++--- modules/server/src/network/resolvers/index.ts | 5 +++-- .../server/src/network/resolvers/remoteConnections.ts | 5 +++-- modules/server/src/network/setup/fields.ts | 8 +++----- modules/server/src/network/setup/query.ts | 8 ++++---- modules/server/src/network/setup/types.ts | 7 +++++++ modules/server/src/network/types.ts | 2 +- modules/server/src/network/util.ts | 1 - 9 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 modules/server/src/network/setup/types.ts diff --git a/modules/server/src/network/index.ts b/modules/server/src/network/index.ts index c604ebd74..40dfdaedd 100644 --- a/modules/server/src/network/index.ts +++ b/modules/server/src/network/index.ts @@ -4,7 +4,7 @@ import { createResolvers } from './resolvers'; import { getAllFieldTypes } from './setup/fields'; import { fetchRemoteSchemas } from './setup/query'; import { createTypeDefs } from './typeDefs'; -import { NetworkAggregationConfigInput } from './types'; +import { NetworkConfig } from './types'; /** * GQL Federated Search schema setup @@ -16,7 +16,7 @@ import { NetworkAggregationConfigInput } from './types'; export const createSchemaFromNetworkConfig = async ({ networkConfigs, }: { - networkConfigs: NetworkAggregationConfigInput[]; + networkConfigs: NetworkConfig[]; }) => { const networkFields = await fetchRemoteSchemas({ networkConfigs, @@ -25,7 +25,8 @@ export const createSchemaFromNetworkConfig = async ({ const networkFieldTypes = getAllFieldTypes(networkFields, SUPPORTED_AGGREGATIONS_LIST); const typeDefs = createTypeDefs(networkFieldTypes); - const resolvers = createResolvers(configs); + + const resolvers = createResolvers(networkConfigs); const networkSchema = makeExecutableSchema({ typeDefs, resolvers }); diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index ef74c3bde..617a6b6b2 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -7,6 +7,10 @@ import { failure, isSuccess, success } from '../httpResponses'; import { supportedAggregationQueries } from '../queries'; import { NetworkAggregation, NetworkAggregationConfig } from '../types'; import { ASTtoString } from '../util'; +import { supportedAggregationQueries } from '../queries'; +import { NetworkConfig } from '../setup/types'; +import { NetworkAggregation } from '../types'; +import { ASTtoString, fulfilledPromiseFilter } from '../util'; import { CONNECTION_STATUS, RemoteConnection } from './remoteConnections'; /** @@ -118,7 +122,7 @@ type NetworkQuery = { * @param fieldName * @returns */ -const findMatchedAggregationField = (config: NetworkAggregationConfig, fieldName: string) => { +const findMatchedAggregationField = (config: NetworkConfig, fieldName: string) => { return config.supportedAggregations.find((agg) => agg.name === fieldName); }; @@ -127,7 +131,7 @@ const findMatchedAggregationField = (config: NetworkAggregationConfig, fieldName * * @param requestedAggregations */ -const createGqlFieldsString = (config: NetworkAggregationConfig, requestedAggregations: any[]) => { +const createGqlFieldsString = (config: NetworkConfig, requestedAggregations: any[]) => { return requestedAggregations.reduce((gqlString, fieldName) => { const matchedAggregationField = findMatchedAggregationField(config, fieldName); if (matchedAggregationField) { @@ -168,7 +172,7 @@ const createGqlFieldsString = (config: NetworkAggregationConfig, requestedAggreg * @returns */ export const createNetworkQueries = ( - configs: NetworkAggregationConfig[], + configs: NetworkConfig[], requestedAggregations: string[], ): NetworkQuery[] => { // TODO: what if there are no matched findMatchedAggregationField diff --git a/modules/server/src/network/resolvers/index.ts b/modules/server/src/network/resolvers/index.ts index acae11cba..f1f7c9ad5 100644 --- a/modules/server/src/network/resolvers/index.ts +++ b/modules/server/src/network/resolvers/index.ts @@ -1,5 +1,6 @@ import { type GraphQLResolveInfo } from 'graphql'; -import { NetworkAggregationConfig, RemoteConnectionData } from '../types'; +import { NetworkConfig } from '../setup/types'; +import { RemoteConnectionData } from '../types'; import { getRequestedFields } from '../util'; import { aggregationPipeline, createNetworkQueries } from './aggregations'; import { createResponse } from './response'; @@ -19,7 +20,7 @@ type NetworkSearchRoot = { * @param networkFieldTypes * @returns */ -export const createResolvers = (configs: NetworkAggregationConfig[]) => { +export const createResolvers = (configs: NetworkConfig[]) => { return { Query: { network: async ( diff --git a/modules/server/src/network/resolvers/remoteConnections.ts b/modules/server/src/network/resolvers/remoteConnections.ts index 51c594795..f356cafe7 100644 --- a/modules/server/src/network/resolvers/remoteConnections.ts +++ b/modules/server/src/network/resolvers/remoteConnections.ts @@ -1,6 +1,7 @@ -import { ConnectionStatus, NetworkAggregationConfig, RemoteConnectionData } from '@/network/types'; +import { ConnectionStatus, RemoteConnectionData } from '@/network/types'; import axios from 'axios'; import urljoin from 'url-join'; +import { NetworkConfig } from '../setup/types'; /* * Setup config connections @@ -51,7 +52,7 @@ export const CONNECTION_STATUS = { * @returns remote connection metadata */ export const resolveRemoteConnectionNodes = async ( - networkConfigs: NetworkAggregationConfig[], + networkConfigs: NetworkConfig[], ): Promise => { /** * Promise.all is safe because we handle errors in checkRemoteConnectionStatus diff --git a/modules/server/src/network/setup/fields.ts b/modules/server/src/network/setup/fields.ts index bb68d7ea6..bfce81e5e 100644 --- a/modules/server/src/network/setup/fields.ts +++ b/modules/server/src/network/setup/fields.ts @@ -1,6 +1,7 @@ import { SupportedAggregation } from '../common'; import { GQLFieldType } from '../queries'; -import { NetworkAggregationConfig, NetworkFieldType, SupportedNetworkFieldType } from '../types'; +import { NetworkFieldType, SupportedNetworkFieldType } from '../types'; +import { NetworkConfig } from './types'; export type NetworkFields = { name: string; fields: GQLFieldType[] }; @@ -22,10 +23,7 @@ export const getFieldTypes = ( ) => { const fieldTypes = fields.reduce( ( - aggregations: Pick< - NetworkAggregationConfig, - 'supportedAggregations' | 'unsupportedAggregations' - >, + aggregations: Pick, field, ) => { const fieldType = field.type.name; diff --git a/modules/server/src/network/setup/query.ts b/modules/server/src/network/setup/query.ts index 7468e5771..5b2f7e1b8 100644 --- a/modules/server/src/network/setup/query.ts +++ b/modules/server/src/network/setup/query.ts @@ -1,12 +1,12 @@ import { NetworkAggregationError } from '../errors'; import { fetchGql } from '../gql'; import { gqlAggregationTypeQuery, GQLTypeQueryResponse } from '../queries'; -import { NetworkAggregationConfigInput } from '../types'; +import { NetworkConfig } from '../types'; import { fulfilledPromiseFilter } from '../util'; import { NetworkFields } from './fields'; type NetworkQueryResult = PromiseFulfilledResult<{ - config: NetworkAggregationConfigInput; + config: NetworkConfig; gqlResponse: GQLTypeQueryResponse; }>; @@ -23,7 +23,7 @@ type NetworkQueryResult = PromiseFulfilledResult<{ * @throws Unexpected data error */ const fetchRemoteSchema = async ( - config: NetworkAggregationConfigInput, + config: NetworkConfig, ): Promise => { const { graphqlUrl, documentType } = config; try { @@ -57,7 +57,7 @@ const fetchRemoteSchema = async ( export const fetchRemoteSchemas = async ({ networkConfigs, }: { - networkConfigs: NetworkAggregationConfigInput[]; + networkConfigs: NetworkConfig[]; }): Promise => { // query remote connection types const networkQueryPromises = networkConfigs.map(async (config) => { diff --git a/modules/server/src/network/setup/types.ts b/modules/server/src/network/setup/types.ts new file mode 100644 index 000000000..53765628a --- /dev/null +++ b/modules/server/src/network/setup/types.ts @@ -0,0 +1,7 @@ +// environment config +export type NetworkConfig = { + graphqlUrl: string; + documentType: string; + documentName: string; + displayName: string; +}; diff --git a/modules/server/src/network/types.ts b/modules/server/src/network/types.ts index 626678488..2c0706e0b 100644 --- a/modules/server/src/network/types.ts +++ b/modules/server/src/network/types.ts @@ -25,7 +25,7 @@ export type SupportedNetworkFieldType = NetworkFieldType; * This includes the original config information plus computed fields * that are generated in the network search initialization process. */ -export type NetworkAggregationConfig = NetworkAggregationConfigInput & { +export type NetworkAggregationConfig = NetworkConfig & { supportedAggregations: SupportedNetworkFieldType[]; unsupportedAggregations: NetworkFieldType[]; }; diff --git a/modules/server/src/network/util.ts b/modules/server/src/network/util.ts index d279a013e..8eb4a0c4f 100644 --- a/modules/server/src/network/util.ts +++ b/modules/server/src/network/util.ts @@ -1,6 +1,5 @@ import { DocumentNode, GraphQLResolveInfo, print } from 'graphql'; import graphqlFields from 'graphql-fields'; -import { NetworkAggregationConfig, SupportedNetworkFieldType } from './types'; /* * GraphQL AST => String From 9412feda67d1c74a57e884994f2fe267d5499952 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Thu, 19 Sep 2024 01:17:23 -0400 Subject: [PATCH 05/11] type cleanup --- modules/server/src/network/index.ts | 4 ++-- .../server/src/network/resolvers/aggregations.ts | 5 ++--- modules/server/src/network/setup/fields.ts | 16 ++++++++++++---- modules/server/src/network/setup/query.ts | 2 +- modules/server/src/network/types.ts | 11 ++--------- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/modules/server/src/network/index.ts b/modules/server/src/network/index.ts index 40dfdaedd..b6ec49e41 100644 --- a/modules/server/src/network/index.ts +++ b/modules/server/src/network/index.ts @@ -1,10 +1,10 @@ import { makeExecutableSchema } from '@graphql-tools/schema'; -import { SUPPORTED_AGGREGATIONS, SUPPORTED_AGGREGATIONS_LIST } from './common'; +import { SUPPORTED_AGGREGATIONS_LIST } from './common'; import { createResolvers } from './resolvers'; import { getAllFieldTypes } from './setup/fields'; import { fetchRemoteSchemas } from './setup/query'; +import { NetworkConfig } from './setup/types'; import { createTypeDefs } from './typeDefs'; -import { NetworkConfig } from './types'; /** * GQL Federated Search schema setup diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index 617a6b6b2..9f6a61a60 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -115,8 +115,7 @@ type NetworkQuery = { }; /** - * Find requested field in remote connection supported fields - * all nodes may not have all fields + * Find requested field in remote connection supported fields all nodes may not have all fields * * @param config * @param fieldName @@ -127,7 +126,7 @@ const findMatchedAggregationField = (config: NetworkConfig, fieldName: string) = }; /** - * construct gql string { [fieldName] { [Aggregation query fields] } } + * Construct gql string { [fieldName] { [Aggregation query fields] } } based on requested fields * * @param requestedAggregations */ diff --git a/modules/server/src/network/setup/fields.ts b/modules/server/src/network/setup/fields.ts index bfce81e5e..85f14d5ed 100644 --- a/modules/server/src/network/setup/fields.ts +++ b/modules/server/src/network/setup/fields.ts @@ -1,7 +1,11 @@ import { SupportedAggregation } from '../common'; import { GQLFieldType } from '../queries'; -import { NetworkFieldType, SupportedNetworkFieldType } from '../types'; -import { NetworkConfig } from './types'; +import { + NetworkFieldType, + SupportedAggregations, + SupportedNetworkFieldType, + UnsupportedAggregations, +} from '../types'; export type NetworkFields = { name: string; fields: GQLFieldType[] }; @@ -13,7 +17,8 @@ const isSupportedType = ( }; /** - * Reduce response into simplified object, returning both supported and unsupported types + * Parse network fields into supporter and unsupported + * * @param fields * @returns { supportedAggregations: [], unsupportedAggregations: [] } */ @@ -23,7 +28,10 @@ export const getFieldTypes = ( ) => { const fieldTypes = fields.reduce( ( - aggregations: Pick, + aggregations: { + supportedAggregations: SupportedAggregations; + unsupportedAggregations: UnsupportedAggregations; + }, field, ) => { const fieldType = field.type.name; diff --git a/modules/server/src/network/setup/query.ts b/modules/server/src/network/setup/query.ts index 5b2f7e1b8..543981254 100644 --- a/modules/server/src/network/setup/query.ts +++ b/modules/server/src/network/setup/query.ts @@ -1,9 +1,9 @@ import { NetworkAggregationError } from '../errors'; import { fetchGql } from '../gql'; import { gqlAggregationTypeQuery, GQLTypeQueryResponse } from '../queries'; -import { NetworkConfig } from '../types'; import { fulfilledPromiseFilter } from '../util'; import { NetworkFields } from './fields'; +import { NetworkConfig } from './types'; type NetworkQueryResult = PromiseFulfilledResult<{ config: NetworkConfig; diff --git a/modules/server/src/network/types.ts b/modules/server/src/network/types.ts index 2c0706e0b..bd9c71446 100644 --- a/modules/server/src/network/types.ts +++ b/modules/server/src/network/types.ts @@ -20,15 +20,8 @@ export type NetworkFieldType = { export type SupportedNetworkFieldType = NetworkFieldType; -/** - * Complete aggregation config that defines the network search setup. - * This includes the original config information plus computed fields - * that are generated in the network search initialization process. - */ -export type NetworkAggregationConfig = NetworkConfig & { - supportedAggregations: SupportedNetworkFieldType[]; - unsupportedAggregations: NetworkFieldType[]; -}; +export type SupportedAggregations = SupportedNetworkFieldType[]; +export type UnsupportedAggregations = NetworkFieldType[]; export type ConnectionStatus = ObjectValues; From f764bd0c52080f9e9af039ec99f78f6972c0996a Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Thu, 19 Sep 2024 11:52:04 -0400 Subject: [PATCH 06/11] only send network queries with original query fields --- .../src/network/resolvers/aggregations.ts | 62 +++++++++++-------- modules/server/src/network/resolvers/index.ts | 9 +-- modules/server/src/network/util.ts | 17 +++-- 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index 9f6a61a60..93b5dbbad 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -10,7 +10,7 @@ import { ASTtoString } from '../util'; import { supportedAggregationQueries } from '../queries'; import { NetworkConfig } from '../setup/types'; import { NetworkAggregation } from '../types'; -import { ASTtoString, fulfilledPromiseFilter } from '../util'; +import { ASTtoString } from '../util'; import { CONNECTION_STATUS, RemoteConnection } from './remoteConnections'; /** @@ -126,22 +126,41 @@ const findMatchedAggregationField = (config: NetworkConfig, fieldName: string) = }; /** - * Construct gql string { [fieldName] { [Aggregation query fields] } } based on requested fields * - * @param requestedAggregations + * @param requestedFields + * + * @example + * ### Input + * ``` + * { donors: { + * buckets: { + * bucket_count: {}, + * } + * }} + * ``` + * + * ### Output + * ``` + * ` + * { donors { + * buckets { + * bucket_count + * } + * }} + * ` + * ``` */ -const createGqlFieldsString = (config: NetworkConfig, requestedAggregations: any[]) => { - return requestedAggregations.reduce((gqlString, fieldName) => { - const matchedAggregationField = findMatchedAggregationField(config, fieldName); - if (matchedAggregationField) { - const { name, type } = matchedAggregationField; - // get gql query string for supported aggregation - // TODO: only query requested fields + bucket_count if nodes is requested - const aggregationFieldQueryString = supportedAggregationQueries.get(type); - return gqlString + `${name}${aggregationFieldQueryString}`; - } - return gqlString; - }, ''); +const createGqlFieldsString = (requestedFields, documentName) => { + const gqlFieldsString = JSON.stringify(requestedFields) + .replaceAll('"', '') + .replaceAll(':', '') + .replaceAll('{}', '') + .replaceAll(',', ' '); + + // add top level field for query and format with correct brackets + const gqlString = `{${documentName} ${gqlFieldsString}}`; + + return gqlString; }; /** @@ -167,20 +186,13 @@ const createGqlFieldsString = (config: NetworkAggregationConfig, requestedAggreg * Create queries for remote nodes based on requested fields * * @param configs - * @param requestedAggregations + * @param requestedFields * @returns */ -export const createNetworkQueries = ( - configs: NetworkConfig[], - requestedAggregations: string[], -): NetworkQuery[] => { - // TODO: what if there are no matched findMatchedAggregationField +export const createNetworkQueries = (configs: NetworkConfig[], requestedFields): NetworkQuery[] => { const queries = configs .map((config) => { - const gqlStringFields = createGqlFieldsString(config, requestedAggregations); - - // add top level field for query and format with correct brackets - const gqlString = `{${config.documentName} { ${gqlStringFields} }}`; + const gqlString = createGqlFieldsString(requestedFields, config.documentName); /* * convert string to AST object to use as query diff --git a/modules/server/src/network/resolvers/index.ts b/modules/server/src/network/resolvers/index.ts index f1f7c9ad5..9cb89d823 100644 --- a/modules/server/src/network/resolvers/index.ts +++ b/modules/server/src/network/resolvers/index.ts @@ -1,7 +1,7 @@ import { type GraphQLResolveInfo } from 'graphql'; import { NetworkConfig } from '../setup/types'; import { RemoteConnectionData } from '../types'; -import { getRequestedFields } from '../util'; +import { resolveInfoToMap } from '../util'; import { aggregationPipeline, createNetworkQueries } from './aggregations'; import { createResponse } from './response'; @@ -29,13 +29,14 @@ export const createResolvers = (configs: NetworkConfig[]) => { context: unknown, info: GraphQLResolveInfo, ) => { - const { requestedAggregations } = getRequestedFields(info); - const networkQueries = createNetworkQueries(configs, requestedAggregations); + const requestedFieldsMap = resolveInfoToMap(info, 'aggregations'); + const networkQueries = createNetworkQueries(configs, requestedFieldsMap); // Query remote connections and aggregate results + const requestedFields = Object.keys(requestedFieldsMap); const { aggregationResults, nodeInfo } = await aggregationPipeline( networkQueries, - requestedAggregations, + requestedFields, ); const response = createResponse({ aggregationResults, nodeInfo }); diff --git a/modules/server/src/network/util.ts b/modules/server/src/network/util.ts index 8eb4a0c4f..aba6011f1 100644 --- a/modules/server/src/network/util.ts +++ b/modules/server/src/network/util.ts @@ -21,12 +21,21 @@ export const fulfilledPromiseFilter = (result: unknown): result is Resul }; /** - * Returns requested fields + * Turns GraphQLResolveInfo into a map of the requested fields * * @param info GQL request info object - * @returns */ -export const getRequestedFields = (info: GraphQLResolveInfo) => { +export const resolveInfoToMap = (info: GraphQLResolveInfo, key: string) => { const requestedFields = graphqlFields(info); - return { requestedAggregations: Object.keys(requestedFields.aggregations) }; + const aggregations = requestedFields[key]; + + // ensure __typename will be queried to network nodes + const aggs = Object.keys(aggregations).reduce((aggs, key) => { + const element = aggregations[key]; + if (!element.hasOwnProperty('__typename')) { + element['__typename'] = {}; + } + return { ...aggs, [key]: element }; + }, {}); + return aggs; }; From 7d1a46b86dfe7528af941d8737d89cdc2f8e57e5 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Thu, 19 Sep 2024 13:26:04 -0400 Subject: [PATCH 07/11] move gql health check into module --- modules/server/src/network/health.ts | 36 +++++++++ .../network/resolvers/remoteConnections.ts | 73 ------------------- 2 files changed, 36 insertions(+), 73 deletions(-) create mode 100644 modules/server/src/network/health.ts diff --git a/modules/server/src/network/health.ts b/modules/server/src/network/health.ts new file mode 100644 index 000000000..6373d314e --- /dev/null +++ b/modules/server/src/network/health.ts @@ -0,0 +1,36 @@ +import axios from 'axios'; +import urljoin from 'url-join'; +import { CONNECTION_STATUS } from './resolvers/nodes'; +import { ConnectionStatus } from './types'; + +/** + * Check the status of remote connections + * + * @param url - Remote connection url + * @returns A connection status + */ +export const gqlHealthCheck = async (url: string): Promise => { + /* + * recommended way to health check gql server is to run the `__typename` query that every server has + * very small query with no additional params to, so using GET is not a concern for the GQL server + * adds recommended pre flight header to make sure Apollo doesn't block request by CSRF protection + */ + const healthCheckQuery = '?query=%7B__typename%7D'; + const fullUrl = urljoin(url, healthCheckQuery); + + try { + const pong = await axios.get(fullUrl, { + headers: { 'apollo-require-preflight': 'true', 'Content-Type': 'application/json' }, + }); + + // only need to check the server returns successfully (just a health check) + if (pong.data.data?.__typename) { + return CONNECTION_STATUS.OK; + } else { + throw Error('no data object returned from GQL server __typname query'); + } + } catch (error) { + console.error(error); + return CONNECTION_STATUS.ERROR; + } +}; diff --git a/modules/server/src/network/resolvers/remoteConnections.ts b/modules/server/src/network/resolvers/remoteConnections.ts index f356cafe7..58d9cbc70 100644 --- a/modules/server/src/network/resolvers/remoteConnections.ts +++ b/modules/server/src/network/resolvers/remoteConnections.ts @@ -1,81 +1,8 @@ -import { ConnectionStatus, RemoteConnectionData } from '@/network/types'; -import axios from 'axios'; -import urljoin from 'url-join'; -import { NetworkConfig } from '../setup/types'; - -/* - * Setup config connections - */ - -/** - * Check the status of remote connections - * - * @param url - Remote connection url - * @returns A connection status - */ -const gqlHealthCheck = async (url: string): Promise => { - /* - * recommended way to health check gql server is to run the `__typename` query that every server has - * very small query with no additional params to, so using GET is not a concern for the GQL server - * adds recommended pre flight header to make sure Apollo doesn't block request by CSRF protection - */ - const healthCheckQuery = '?query=%7B__typename%7D'; - const fullUrl = urljoin(url, healthCheckQuery); - - try { - const pong = await axios.get(fullUrl, { - headers: { 'apollo-require-preflight': 'true', 'Content-Type': 'application/json' }, - }); - - // only need to check the server returns successfully (just a health check) - if (pong.data.data?.__typename) { - return CONNECTION_STATUS.OK; - } else { - throw Error('no data object returned from GQL server __typname query'); - } - } catch (error) { - console.error(error); - return CONNECTION_STATUS.ERROR; - } -}; - export const CONNECTION_STATUS = { OK: 'OK', ERROR: 'ERROR', } as const; -/** - * Returns available remote configuration data from in memory config - * Adds connection status data, and available GQL types from the remote servers - * - * @param networkConfigs - * @returns remote connection metadata - */ -export const resolveRemoteConnectionNodes = async ( - networkConfigs: NetworkConfig[], -): Promise => { - /** - * Promise.all is safe because we handle errors in checkRemoteConnectionStatus - */ - return await Promise.all( - networkConfigs.map(async (config) => { - const status = await gqlHealthCheck(config.graphqlUrl); - - const remoteConnectionData: RemoteConnectionData = { - url: config.graphqlUrl, - name: config.displayName, - description: '', - documentName: config.documentType, - availableAggregations: config.supportedAggregations, - status, - totalHits: 0, - errors: [], - }; - return remoteConnectionData; - }), - ); -}; - /* * Querying from resolvers remote connections */ From 50461312470dadc62b1b2fd957901d9a4414c182 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Thu, 19 Sep 2024 14:36:01 -0400 Subject: [PATCH 08/11] cleanup, renaming types, removing redundant code --- modules/server/src/network/health.ts | 1 - modules/server/src/network/index.ts | 2 +- modules/server/src/network/queries/index.ts | 22 +-------------- .../src/network/resolvers/aggregations.ts | 18 +++--------- modules/server/src/network/resolvers/index.ts | 8 +++--- .../{remoteConnections.ts => networkNode.ts} | 2 +- modules/server/src/network/setup/fields.ts | 2 +- modules/server/src/network/setup/query.ts | 2 +- modules/server/src/network/typeDefs/index.ts | 2 +- .../src/network/typeDefs/remoteConnections.ts | 6 +--- .../server/src/network/types/aggregations.ts | 15 ++++++++++ .../{setup/types.ts => types/setup.ts} | 5 +++- .../server/src/network/{ => types}/types.ts | 28 ++----------------- 13 files changed, 37 insertions(+), 76 deletions(-) rename modules/server/src/network/resolvers/{remoteConnections.ts => networkNode.ts} (87%) create mode 100644 modules/server/src/network/types/aggregations.ts rename modules/server/src/network/{setup/types.ts => types/setup.ts} (78%) rename modules/server/src/network/{ => types}/types.ts (59%) diff --git a/modules/server/src/network/health.ts b/modules/server/src/network/health.ts index 6373d314e..0a40f7a39 100644 --- a/modules/server/src/network/health.ts +++ b/modules/server/src/network/health.ts @@ -1,6 +1,5 @@ import axios from 'axios'; import urljoin from 'url-join'; -import { CONNECTION_STATUS } from './resolvers/nodes'; import { ConnectionStatus } from './types'; /** diff --git a/modules/server/src/network/index.ts b/modules/server/src/network/index.ts index b6ec49e41..a82648023 100644 --- a/modules/server/src/network/index.ts +++ b/modules/server/src/network/index.ts @@ -3,8 +3,8 @@ import { SUPPORTED_AGGREGATIONS_LIST } from './common'; import { createResolvers } from './resolvers'; import { getAllFieldTypes } from './setup/fields'; import { fetchRemoteSchemas } from './setup/query'; -import { NetworkConfig } from './setup/types'; import { createTypeDefs } from './typeDefs'; +import { NetworkConfig } from './types/setup'; /** * GQL Federated Search schema setup diff --git a/modules/server/src/network/queries/index.ts b/modules/server/src/network/queries/index.ts index 99cfe3d2b..5d027bb9c 100644 --- a/modules/server/src/network/queries/index.ts +++ b/modules/server/src/network/queries/index.ts @@ -5,8 +5,7 @@ * These are the GQL queries for the supported GQL types */ -import { SupportedAggregation, SUPPORTED_AGGREGATIONS } from '../common'; -import { Aggregations, NumericAggregations } from '../types'; +import { Aggregations } from '../types/aggregations'; export type GQLFieldType = { name: string; @@ -46,22 +45,3 @@ export const gqlAggregationTypeQuery = `#graphql } } `; - -export const aggregationsQuery = /* GraphQL */ ` - #graphql - { - __typename - bucket_count - buckets { - key - doc_count - } - } -`; - -/** - * Returns gql query for gql type - */ -export const supportedAggregationQueries = new Map(); -supportedAggregationQueries.set(SUPPORTED_AGGREGATIONS.Aggregations, aggregationsQuery); -supportedAggregationQueries.set(SUPPORTED_AGGREGATIONS.NumericAggregations, ''); diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index 93b5dbbad..fab5e4ce8 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -11,7 +11,7 @@ import { supportedAggregationQueries } from '../queries'; import { NetworkConfig } from '../setup/types'; import { NetworkAggregation } from '../types'; import { ASTtoString } from '../util'; -import { CONNECTION_STATUS, RemoteConnection } from './remoteConnections'; +import { CONNECTION_STATUS, NetworkNode } from './networkNode'; /** * Query remote connections and handle network responses @@ -70,12 +70,12 @@ export const aggregationPipeline = async ( return { ...accumulator, [field]: emptyAggregation }; }, {}); - const nodeInfo: any = []; + const nodeInfo: NetworkNode[] = []; const aggregationResultPromises = queries.map< Promise<{ aggregations: any; - remoteConnection: RemoteConnection; + remoteConnection: NetworkNode; }> >(async (query) => { const name = query.url; // TODO: use readable name not url @@ -115,17 +115,7 @@ type NetworkQuery = { }; /** - * Find requested field in remote connection supported fields all nodes may not have all fields - * - * @param config - * @param fieldName - * @returns - */ -const findMatchedAggregationField = (config: NetworkConfig, fieldName: string) => { - return config.supportedAggregations.find((agg) => agg.name === fieldName); -}; - -/** + * Converts info field object into string * * @param requestedFields * diff --git a/modules/server/src/network/resolvers/index.ts b/modules/server/src/network/resolvers/index.ts index 9cb89d823..c4336dc18 100644 --- a/modules/server/src/network/resolvers/index.ts +++ b/modules/server/src/network/resolvers/index.ts @@ -1,12 +1,12 @@ import { type GraphQLResolveInfo } from 'graphql'; -import { NetworkConfig } from '../setup/types'; -import { RemoteConnectionData } from '../types'; +import { NetworkConfig } from '../types/setup'; import { resolveInfoToMap } from '../util'; import { aggregationPipeline, createNetworkQueries } from './aggregations'; +import { NetworkNode } from './networkNode'; import { createResponse } from './response'; -type NetworkSearchRoot = { - nodes: RemoteConnectionData[]; +export type NetworkSearchRoot = { + nodes: NetworkNode[]; aggregations: Record; }; diff --git a/modules/server/src/network/resolvers/remoteConnections.ts b/modules/server/src/network/resolvers/networkNode.ts similarity index 87% rename from modules/server/src/network/resolvers/remoteConnections.ts rename to modules/server/src/network/resolvers/networkNode.ts index 58d9cbc70..1ea67bbfc 100644 --- a/modules/server/src/network/resolvers/remoteConnections.ts +++ b/modules/server/src/network/resolvers/networkNode.ts @@ -7,7 +7,7 @@ export const CONNECTION_STATUS = { * Querying from resolvers remote connections */ -export type RemoteConnection = { +export type NetworkNode = { name: string; count: number; status: keyof typeof CONNECTION_STATUS; diff --git a/modules/server/src/network/setup/fields.ts b/modules/server/src/network/setup/fields.ts index 85f14d5ed..f553adfcb 100644 --- a/modules/server/src/network/setup/fields.ts +++ b/modules/server/src/network/setup/fields.ts @@ -5,7 +5,7 @@ import { SupportedAggregations, SupportedNetworkFieldType, UnsupportedAggregations, -} from '../types'; +} from '../types/types'; export type NetworkFields = { name: string; fields: GQLFieldType[] }; diff --git a/modules/server/src/network/setup/query.ts b/modules/server/src/network/setup/query.ts index 543981254..a8fecce36 100644 --- a/modules/server/src/network/setup/query.ts +++ b/modules/server/src/network/setup/query.ts @@ -1,9 +1,9 @@ import { NetworkAggregationError } from '../errors'; import { fetchGql } from '../gql'; import { gqlAggregationTypeQuery, GQLTypeQueryResponse } from '../queries'; +import { NetworkConfig } from '../types/setup'; import { fulfilledPromiseFilter } from '../util'; import { NetworkFields } from './fields'; -import { NetworkConfig } from './types'; type NetworkQueryResult = PromiseFulfilledResult<{ config: NetworkConfig; diff --git a/modules/server/src/network/typeDefs/index.ts b/modules/server/src/network/typeDefs/index.ts index 879a8bf43..a37d979a6 100644 --- a/modules/server/src/network/typeDefs/index.ts +++ b/modules/server/src/network/typeDefs/index.ts @@ -1,5 +1,5 @@ import { mergeTypeDefs } from '@graphql-tools/merge'; -import { SupportedNetworkFieldType } from '../types'; +import { SupportedNetworkFieldType } from '../types/types'; import { createNetworkAggregationTypeDefs } from './aggregations'; import { remoteConnectionTypes } from './remoteConnections'; diff --git a/modules/server/src/network/typeDefs/remoteConnections.ts b/modules/server/src/network/typeDefs/remoteConnections.ts index 350efc648..946400872 100644 --- a/modules/server/src/network/typeDefs/remoteConnections.ts +++ b/modules/server/src/network/typeDefs/remoteConnections.ts @@ -4,12 +4,8 @@ export const remoteConnectionTypes = `#graphql } type RemoteConnectionNode { - url: String name: String - description: String - documentName: String - availableAggregations: [String] - totalHits: Int + count: Int status: String errors: [String] } diff --git a/modules/server/src/network/types/aggregations.ts b/modules/server/src/network/types/aggregations.ts new file mode 100644 index 000000000..0411e2554 --- /dev/null +++ b/modules/server/src/network/types/aggregations.ts @@ -0,0 +1,15 @@ +/* + * Typescript types for aggregations + */ + +export type Bucket = { + doc_count: number; + key: string; +}; + +export type Aggregations = { + bucket_count: number; + buckets: Bucket[]; +}; + +export type NumericAggregations = {}; diff --git a/modules/server/src/network/setup/types.ts b/modules/server/src/network/types/setup.ts similarity index 78% rename from modules/server/src/network/setup/types.ts rename to modules/server/src/network/types/setup.ts index 53765628a..5176a7d0a 100644 --- a/modules/server/src/network/setup/types.ts +++ b/modules/server/src/network/types/setup.ts @@ -1,4 +1,7 @@ -// environment config +/* + * Setup of network types + */ + export type NetworkConfig = { graphqlUrl: string; documentType: string; diff --git a/modules/server/src/network/types.ts b/modules/server/src/network/types/types.ts similarity index 59% rename from modules/server/src/network/types.ts rename to modules/server/src/network/types/types.ts index bd9c71446..0cf4a4f3f 100644 --- a/modules/server/src/network/types.ts +++ b/modules/server/src/network/types/types.ts @@ -1,6 +1,7 @@ import { ObjectValues } from '@/utils/types'; -import { SupportedAggregation } from './common'; -import { CONNECTION_STATUS } from './resolvers/remoteConnections'; +import { SupportedAggregation } from '../common'; +import { CONNECTION_STATUS } from '../resolvers/networkNode'; +import { Aggregations, Bucket, NumericAggregations } from './aggregations'; // environment config export type NetworkAggregationConfigInput = { @@ -25,29 +26,6 @@ export type UnsupportedAggregations = NetworkFieldType[]; export type ConnectionStatus = ObjectValues; -export type RemoteConnectionData = { - url: string; - name: string; - description: string; - documentName: string; - availableAggregations: SupportedNetworkFieldType[]; - totalHits: number; - errors: string[]; - status: ConnectionStatus; -}; - -export type Bucket = { - doc_count: number; - key: string; -}; - -export type Aggregations = { - bucket_count: number; - buckets: Bucket[]; -}; - -export type NumericAggregations = {}; - export type RemoteAggregation = { [key: string]: Aggregations | NumericAggregations }; export type NetworkAggregation = { From c11dd43e2d66eeadb70aacd0aaf8586b74f7d440 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Thu, 19 Sep 2024 14:54:09 -0400 Subject: [PATCH 09/11] add type --- modules/server/src/network/resolvers/aggregations.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index fab5e4ce8..ddb240457 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -140,7 +140,7 @@ type NetworkQuery = { * ` * ``` */ -const createGqlFieldsString = (requestedFields, documentName) => { +const createGqlFieldsString = (requestedFields: {}, documentName: string) => { const gqlFieldsString = JSON.stringify(requestedFields) .replaceAll('"', '') .replaceAll(':', '') @@ -179,7 +179,10 @@ const createGqlFieldsString = (config: NetworkAggregationConfig, requestedAggreg * @param requestedFields * @returns */ -export const createNetworkQueries = (configs: NetworkConfig[], requestedFields): NetworkQuery[] => { +export const createNetworkQueries = ( + configs: NetworkConfig[], + requestedFields: {}, +): NetworkQuery[] => { const queries = configs .map((config) => { const gqlString = createGqlFieldsString(requestedFields, config.documentName); From 68ab5876337bbffdf79c2adc987f4b5e8044f9d0 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Fri, 20 Sep 2024 13:23:23 -0400 Subject: [PATCH 10/11] fix merged conflict --- modules/server/src/network/health.ts | 3 ++- .../src/network/resolvers/aggregations.ts | 26 +------------------ 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/modules/server/src/network/health.ts b/modules/server/src/network/health.ts index 0a40f7a39..37a2dde77 100644 --- a/modules/server/src/network/health.ts +++ b/modules/server/src/network/health.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import urljoin from 'url-join'; -import { ConnectionStatus } from './types'; +import { CONNECTION_STATUS } from './resolvers/networkNode'; +import { ConnectionStatus } from './types/types'; /** * Check the status of remote connections diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index ddb240457..19f2df7e7 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -4,12 +4,7 @@ import { DocumentNode } from 'graphql'; import { resolveAggregations } from '../aggregations'; import { fetchGql } from '../gql'; import { failure, isSuccess, success } from '../httpResponses'; -import { supportedAggregationQueries } from '../queries'; -import { NetworkAggregation, NetworkAggregationConfig } from '../types'; -import { ASTtoString } from '../util'; -import { supportedAggregationQueries } from '../queries'; -import { NetworkConfig } from '../setup/types'; -import { NetworkAggregation } from '../types'; +import { NetworkAggregation } from '../types/types'; import { ASTtoString } from '../util'; import { CONNECTION_STATUS, NetworkNode } from './networkNode'; @@ -153,25 +148,6 @@ const createGqlFieldsString = (requestedFields: {}, documentName: string) => { return gqlString; }; -/** - * Create queries for remote nodes based on requested fields - * - * @param requestedAggregations - */ -const createGqlFieldsString = (config: NetworkAggregationConfig, requestedAggregations: any[]) => { - return requestedAggregations.reduce((gqlString, fieldName) => { - const matchedAggregationField = findMatchedAggregationField(config, fieldName); - if (matchedAggregationField) { - const { name, type } = matchedAggregationField; - // get gql query string for supported aggregation - // TODO: only query requested fields + bucket_count if nodes is requested - const aggregationFieldQueryString = supportedAggregationQueries.get(type); - return gqlString + `${name}${aggregationFieldQueryString}`; - } - return gqlString; - }, ''); -}; - /** * Create queries for remote nodes based on requested fields * From f0ddc0cb8586fff58ae5a8f0763a47d33cefefa4 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Fri, 20 Sep 2024 14:30:33 -0400 Subject: [PATCH 11/11] adds full stop. fixes comment typo --- modules/server/src/network/health.ts | 6 +++--- modules/server/src/network/setup/fields.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/server/src/network/health.ts b/modules/server/src/network/health.ts index 37a2dde77..c558501b3 100644 --- a/modules/server/src/network/health.ts +++ b/modules/server/src/network/health.ts @@ -11,9 +11,9 @@ import { ConnectionStatus } from './types/types'; */ export const gqlHealthCheck = async (url: string): Promise => { /* - * recommended way to health check gql server is to run the `__typename` query that every server has - * very small query with no additional params to, so using GET is not a concern for the GQL server - * adds recommended pre flight header to make sure Apollo doesn't block request by CSRF protection + * recommended way to health check gql server is to run the `__typename` query that every server has. + * very small query with no additional params, so using GET is not a concern for the GQL server. + * adds recommended pre flight header to make sure Apollo doesn't block request by CSRF protection. */ const healthCheckQuery = '?query=%7B__typename%7D'; const fullUrl = urljoin(url, healthCheckQuery); diff --git a/modules/server/src/network/setup/fields.ts b/modules/server/src/network/setup/fields.ts index f553adfcb..627bef5bd 100644 --- a/modules/server/src/network/setup/fields.ts +++ b/modules/server/src/network/setup/fields.ts @@ -17,7 +17,7 @@ const isSupportedType = ( }; /** - * Parse network fields into supporter and unsupported + * Parse network fields into supported and unsupported * * @param fields * @returns { supportedAggregations: [], unsupportedAggregations: [] }