From 58e8e0b22611c643c6d5fc9569d9528ea271010a Mon Sep 17 00:00:00 2001 From: chenxsan Date: Fri, 27 Nov 2020 13:14:54 +0800 Subject: [PATCH 1/7] use api key for opencollective --- .github/workflows/testing.yml | 1 + src/utilities/fetch-supporters.js | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2efe1cb35b84..6c0c7e32d1a9 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -87,6 +87,7 @@ jobs: run: yarn build env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + OPENCOLLECTIVEAPIKEY: ${{secrets.OPENCOLLECTIVEAPIKEY}} - run: yarn lint:links e2e: diff --git a/src/utilities/fetch-supporters.js b/src/utilities/fetch-supporters.js index d94345920313..5e625ee875a7 100644 --- a/src/utilities/fetch-supporters.js +++ b/src/utilities/fetch-supporters.js @@ -11,7 +11,11 @@ const REQUIRED_KEYS = ['totalDonations', 'slug', 'name']; const filename = '_supporters.json'; const absoluteFilename = path.resolve(__dirname, '..', 'components', 'Support', filename); -const graphqlEndpoint = 'https://api.opencollective.com/graphql/v2'; +let graphqlEndpoint = 'https://api.opencollective.com/graphql/v2'; + +if (process.env && process.env.OPENCOLLECTIVEAPIKEY) { + graphqlEndpoint = graphqlEndpoint + `/${process.env.OPENCOLLECTIVEAPIKEY}`; +} const membersGraphqlQuery = `query account($limit: Int, $offset: Int) { account(slug: "webpack") { From e767fe1e0aa6a7690846060bc9d199537343debf Mon Sep 17 00:00:00 2001 From: chenxsan Date: Fri, 27 Nov 2020 13:26:30 +0800 Subject: [PATCH 2/7] remove from testing action --- .github/workflows/testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6c0c7e32d1a9..2efe1cb35b84 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -87,7 +87,6 @@ jobs: run: yarn build env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - OPENCOLLECTIVEAPIKEY: ${{secrets.OPENCOLLECTIVEAPIKEY}} - run: yarn lint:links e2e: From 3400d69ae57b19064cb6a0832d3ba831caf16c9a Mon Sep 17 00:00:00 2001 From: chenxsan Date: Fri, 27 Nov 2020 13:28:06 +0800 Subject: [PATCH 3/7] add OPENCOLLECTIVE_API_KEY --- .github/workflows/deploy.yml | 1 + src/utilities/fetch-supporters.js | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 397bc1c9a324..934049561e35 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -42,6 +42,7 @@ jobs: uses: JamesIves/github-pages-deploy-action@3.7.1 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENCOLLECTIVE_API_KEY: ${{ secrets.OPENCOLLECTIVE_API_KEY }} BRANCH: gh-pages FOLDER: dist CLEAN: true diff --git a/src/utilities/fetch-supporters.js b/src/utilities/fetch-supporters.js index 5e625ee875a7..ee55efd3024e 100644 --- a/src/utilities/fetch-supporters.js +++ b/src/utilities/fetch-supporters.js @@ -12,9 +12,12 @@ const filename = '_supporters.json'; const absoluteFilename = path.resolve(__dirname, '..', 'components', 'Support', filename); let graphqlEndpoint = 'https://api.opencollective.com/graphql/v2'; +let graphqlPageSize = 1000; -if (process.env && process.env.OPENCOLLECTIVEAPIKEY) { - graphqlEndpoint = graphqlEndpoint + `/${process.env.OPENCOLLECTIVEAPIKEY}`; +if (process.env && process.env.OPENCOLLECTIVE_API_KEY) { + // use api key when deploying + graphqlEndpoint = graphqlEndpoint + `/${process.env.OPENCOLLECTIVE_API_KEY}`; + graphqlPageSize = 5000; } const membersGraphqlQuery = `query account($limit: Int, $offset: Int) { @@ -55,8 +58,6 @@ const transactionsGraphqlQuery = `query account($limit: Int, $offset: Int) { } }`; -const graphqlPageSize = 5000; - const nodeToSupporter = node => ({ name: node.account.name, slug: node.account.slug, From 4d485853db42e5e47b88245b25a56563c3dbaa14 Mon Sep 17 00:00:00 2001 From: chenxsan Date: Fri, 27 Nov 2020 13:35:50 +0800 Subject: [PATCH 4/7] add comments --- src/utilities/fetch-supporters.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utilities/fetch-supporters.js b/src/utilities/fetch-supporters.js index ee55efd3024e..b47cf3922876 100644 --- a/src/utilities/fetch-supporters.js +++ b/src/utilities/fetch-supporters.js @@ -12,12 +12,14 @@ const filename = '_supporters.json'; const absoluteFilename = path.resolve(__dirname, '..', 'components', 'Support', filename); let graphqlEndpoint = 'https://api.opencollective.com/graphql/v2'; +// defaults to a small size +// see https://github.com/webpack/webpack.js.org/issues/4211 let graphqlPageSize = 1000; if (process.env && process.env.OPENCOLLECTIVE_API_KEY) { // use api key when deploying graphqlEndpoint = graphqlEndpoint + `/${process.env.OPENCOLLECTIVE_API_KEY}`; - graphqlPageSize = 5000; + graphqlPageSize = 5000; // use a large size only for production } const membersGraphqlQuery = `query account($limit: Int, $offset: Int) { From 7a8a864eb94991773e5aa9f02d50202b8e4e3107 Mon Sep 17 00:00:00 2001 From: chenxsan Date: Fri, 27 Nov 2020 13:48:35 +0800 Subject: [PATCH 5/7] make graphqlPageSize smaller --- src/utilities/fetch-supporters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utilities/fetch-supporters.js b/src/utilities/fetch-supporters.js index b47cf3922876..e6a1fe32833c 100644 --- a/src/utilities/fetch-supporters.js +++ b/src/utilities/fetch-supporters.js @@ -14,7 +14,7 @@ const absoluteFilename = path.resolve(__dirname, '..', 'components', 'Support', let graphqlEndpoint = 'https://api.opencollective.com/graphql/v2'; // defaults to a small size // see https://github.com/webpack/webpack.js.org/issues/4211 -let graphqlPageSize = 1000; +let graphqlPageSize = 500; if (process.env && process.env.OPENCOLLECTIVE_API_KEY) { // use api key when deploying From ea86652c3aa11db7bd855bb2efe4aadf2d196833 Mon Sep 17 00:00:00 2001 From: chenxsan Date: Fri, 27 Nov 2020 17:08:32 +0800 Subject: [PATCH 6/7] optimize transaction querying --- src/utilities/fetch-supporters.js | 93 ++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/src/utilities/fetch-supporters.js b/src/utilities/fetch-supporters.js index e6a1fe32833c..7bcc88964127 100644 --- a/src/utilities/fetch-supporters.js +++ b/src/utilities/fetch-supporters.js @@ -9,17 +9,23 @@ const asyncWriteFile = promisify(fs.writeFile); const REQUIRED_KEYS = ['totalDonations', 'slug', 'name']; const filename = '_supporters.json'; -const absoluteFilename = path.resolve(__dirname, '..', 'components', 'Support', filename); - -let graphqlEndpoint = 'https://api.opencollective.com/graphql/v2'; -// defaults to a small size -// see https://github.com/webpack/webpack.js.org/issues/4211 -let graphqlPageSize = 500; - -if (process.env && process.env.OPENCOLLECTIVE_API_KEY) { - // use api key when deploying +const absoluteFilename = path.resolve( + __dirname, + '..', + 'components', + 'Support', + filename +); + +let graphqlEndpoint = + 'https://api.opencollective.com/graphql/v2'; + +// https://github.com/opencollective/opencollective-api/blob/master/server/graphql/v2/query/TransactionsQuery.ts#L81 +const graphqlPageSize = 1000; + +if (process.env && process.env.CI && process.env.OPENCOLLECTIVE_API_KEY) { + // use api key when deploying to production graphqlEndpoint = graphqlEndpoint + `/${process.env.OPENCOLLECTIVE_API_KEY}`; - graphqlPageSize = 5000; // use a large size only for production } const membersGraphqlQuery = `query account($limit: Int, $offset: Int) { @@ -41,10 +47,12 @@ const membersGraphqlQuery = `query account($limit: Int, $offset: Int) { } }`; -const transactionsGraphqlQuery = `query account($limit: Int, $offset: Int) { - account(slug: "webpack") { - transactions(limit: $limit, offset: $offset, includeIncognitoTransactions: false) { - nodes { +// only query transactions in last year +const transactionsGraphqlQuery = `query transactions($dateFrom: ISODateTime, $limit: Int, $offset: Int) { + transactions(account: { + slug: "webpack" + }, dateFrom: $dateFrom, limit: $limit, offset: $offset, includeIncognitoTransactions: false) { + nodes { amountInHostCurrency { value } @@ -56,26 +64,34 @@ const transactionsGraphqlQuery = `query account($limit: Int, $offset: Int) { } createdAt } - } } }`; -const nodeToSupporter = node => ({ +const nodeToSupporter = (node) => ({ name: node.account.name, slug: node.account.slug, website: node.account.website, avatar: node.account.imageUrl, firstDonation: node.createdAt, totalDonations: node.totalDonations.value * 100, - monthlyDonations: 0 + monthlyDonations: 0, }); const getAllNodes = async (graphqlQuery, getNodes) => { const requestOptions = { method: 'POST', uri: graphqlEndpoint, - body: { query: graphqlQuery, variables: { limit: graphqlPageSize, offset: 0 } }, - json: true + body: { + query: graphqlQuery, + variables: { + limit: graphqlPageSize, + offset: 0, + dateFrom: new Date( + new Date().setFullYear(new Date().getFullYear() - 1) + ).toISOString(), // data from last year + }, + }, + json: true, }; let allNodes = []; @@ -89,15 +105,22 @@ const getAllNodes = async (graphqlQuery, getNodes) => { requestOptions.body.variables.offset += graphqlPageSize; if (nodes.length < graphqlPageSize) { return allNodes; + } else { + // sleep for a while + await new Promise((resolve) => setTimeout(resolve, 100)); } } }; -const oneYearAgo = Date.now() - 365 * 24 * 60 * 60 * 1000; (async () => { - const members = await getAllNodes(membersGraphqlQuery, data => data.account.members.nodes); - let supporters = members.map(nodeToSupporter).sort((a, b) => b.totalDonations - a.totalDonations); + const members = await getAllNodes( + membersGraphqlQuery, + (data) => data.account.members.nodes + ); + let supporters = members + .map(nodeToSupporter) + .sort((a, b) => b.totalDonations - a.totalDonations); // Deduplicating supporters with multiple orders supporters = uniqBy(supporters, 'slug'); @@ -106,26 +129,31 @@ const oneYearAgo = Date.now() - 365 * 24 * 60 * 60 * 1000; for (const supporter of supporters) { for (const key of REQUIRED_KEYS) { if (!supporter || typeof supporter !== 'object') { - throw new Error(`Supporters: ${JSON.stringify(supporter)} is not an object.`); + throw new Error( + `Supporters: ${JSON.stringify(supporter)} is not an object.` + ); } if (!(key in supporter)) { - throw new Error(`Supporters: ${JSON.stringify(supporter)} doesn't include ${key}.`); + throw new Error( + `Supporters: ${JSON.stringify(supporter)} doesn't include ${key}.` + ); } } supportersBySlug.set(supporter.slug, supporter); } // Calculate monthly amount from transactions - const transactions = await getAllNodes(transactionsGraphqlQuery, data => data.account.transactions.nodes); + const transactions = await getAllNodes( + transactionsGraphqlQuery, + (data) => data.transactions.nodes + ); for (const transaction of transactions) { if (!transaction.amountInHostCurrency) continue; const amount = transaction.amountInHostCurrency.value; if (!amount || amount <= 0) continue; - const date = +new Date(transaction.createdAt); - if (date < oneYearAgo) continue; const supporter = supportersBySlug.get(transaction.fromAccount.slug); if (!supporter) continue; - supporter.monthlyDonations += amount * 100 / 12; + supporter.monthlyDonations += (amount * 100) / 12; } for (const supporter of supporters) { @@ -133,10 +161,11 @@ const oneYearAgo = Date.now() - 365 * 24 * 60 * 60 * 1000; } // Write the file - return asyncWriteFile(absoluteFilename, JSON.stringify(supporters, null, 2)).then(() => - console.log(`Fetched 1 file: ${filename}`) - ); -})().catch(error => { + return asyncWriteFile( + absoluteFilename, + JSON.stringify(supporters, null, 2) + ).then(() => console.log(`Fetched 1 file: ${filename}`)); +})().catch((error) => { console.error('utilities/fetch-supporters:', error); process.exitCode = 1; }); From 804855d9474075e5f18d4dba2297a323333a4017 Mon Sep 17 00:00:00 2001 From: Sam Chen Date: Sat, 28 Nov 2020 08:38:41 +0800 Subject: [PATCH 7/7] remove api key --- src/utilities/fetch-supporters.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/utilities/fetch-supporters.js b/src/utilities/fetch-supporters.js index 7bcc88964127..e5d55b33c06a 100644 --- a/src/utilities/fetch-supporters.js +++ b/src/utilities/fetch-supporters.js @@ -17,17 +17,12 @@ const absoluteFilename = path.resolve( filename ); -let graphqlEndpoint = +const graphqlEndpoint = 'https://api.opencollective.com/graphql/v2'; // https://github.com/opencollective/opencollective-api/blob/master/server/graphql/v2/query/TransactionsQuery.ts#L81 const graphqlPageSize = 1000; -if (process.env && process.env.CI && process.env.OPENCOLLECTIVE_API_KEY) { - // use api key when deploying to production - graphqlEndpoint = graphqlEndpoint + `/${process.env.OPENCOLLECTIVE_API_KEY}`; -} - const membersGraphqlQuery = `query account($limit: Int, $offset: Int) { account(slug: "webpack") { members(limit: $limit, offset: $offset) {