fix 503 error from opencollective (#4214)

* use api key for opencollective

* remove from testing action

* add OPENCOLLECTIVE_API_KEY

* add comments

* make graphqlPageSize smaller

* optimize transaction querying

* remove api key

Author: chenxsan

.github/workflows/deploy.yml

@@ -42,6 +42,7 @@ jobs:
         uses: JamesIves/[email protected]
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENCOLLECTIVE_API_KEY: ${{ secrets.OPENCOLLECTIVE_API_KEY }}
           BRANCH: gh-pages
           FOLDER: dist
           CLEAN: true

src/utilities/fetch-supporters.js

@@ -9,9 +9,19 @@ const asyncWriteFile = promisify(fs.writeFile);
 
 const REQUIRED_KEYS = ['totalDonations', 'slug', 'name'];
 const filename = '_supporters.json';
-const absoluteFilename = path.resolve(__dirname, '..', 'components', 'Support', filename);
+const absoluteFilename = path.resolve(
+  __dirname,
+  '..',
+  'components',
+  'Support',
+  filename
+);
 
-const graphqlEndpoint = 'https://api.opencollective.com/graphql/v2';
+const graphqlEndpoint =
+  'https://api.opencollective.com/graphql/v2';
+
+// https://github.com/opencollective/opencollective-api/blob/master/server/graphql/v2/query/TransactionsQuery.ts#L81
+const graphqlPageSize = 1000;
 
 const membersGraphqlQuery = `query account($limit: Int, $offset: Int) {
   account(slug: "webpack") {
@@ -32,10 +42,12 @@ const membersGraphqlQuery = `query account($limit: Int, $offset: Int) {
   }
 }`;
 
-const transactionsGraphqlQuery = `query account($limit: Int, $offset: Int) {
-  account(slug: "webpack") {
-    transactions(limit: $limit, offset: $offset, includeIncognitoTransactions: false) {
-      nodes {
+// only query transactions in last year
+const transactionsGraphqlQuery = `query transactions($dateFrom: ISODateTime, $limit: Int, $offset: Int) {
+  transactions(account: {
+    slug: "webpack"
+  }, dateFrom: $dateFrom, limit: $limit, offset: $offset, includeIncognitoTransactions: false) {
+    nodes {
         amountInHostCurrency {
           value
         }
@@ -47,28 +59,34 @@ const transactionsGraphqlQuery = `query account($limit: Int, $offset: Int) {
         }
         createdAt
       }
-    }
   }
 }`;
 
-const graphqlPageSize = 5000;
-
-const nodeToSupporter = node => ({
+const nodeToSupporter = (node) => ({
   name: node.account.name,
   slug: node.account.slug,
   website: node.account.website,
   avatar: node.account.imageUrl,
   firstDonation: node.createdAt,
   totalDonations: node.totalDonations.value * 100,
-  monthlyDonations: 0
+  monthlyDonations: 0,
 });
 
 const getAllNodes = async (graphqlQuery, getNodes) => {
   const requestOptions = {
     method: 'POST',
     uri: graphqlEndpoint,
-    body: { query: graphqlQuery, variables: { limit: graphqlPageSize, offset: 0 } },
-    json: true
+    body: {
+      query: graphqlQuery,
+      variables: {
+        limit: graphqlPageSize,
+        offset: 0,
+        dateFrom: new Date(
+          new Date().setFullYear(new Date().getFullYear() - 1)
+        ).toISOString(), // data from last year
+      },
+    },
+    json: true,
   };
 
   let allNodes = [];
@@ -82,15 +100,22 @@ const getAllNodes = async (graphqlQuery, getNodes) => {
     requestOptions.body.variables.offset += graphqlPageSize;
     if (nodes.length < graphqlPageSize) {
       return allNodes;
+    } else {
+      // sleep for a while
+      await new Promise((resolve) => setTimeout(resolve, 100));
     }
   }
 };
 
-const oneYearAgo = Date.now() - 365 * 24 * 60 * 60 * 1000;
 
 (async () => {
-  const members = await getAllNodes(membersGraphqlQuery, data => data.account.members.nodes);
-  let supporters = members.map(nodeToSupporter).sort((a, b) => b.totalDonations - a.totalDonations);
+  const members = await getAllNodes(
+    membersGraphqlQuery,
+    (data) => data.account.members.nodes
+  );
+  let supporters = members
+    .map(nodeToSupporter)
+    .sort((a, b) => b.totalDonations - a.totalDonations);
 
   // Deduplicating supporters with multiple orders
   supporters = uniqBy(supporters, 'slug');
@@ -99,37 +124,43 @@ const oneYearAgo = Date.now() - 365 * 24 * 60 * 60 * 1000;
   for (const supporter of supporters) {
     for (const key of REQUIRED_KEYS) {
       if (!supporter || typeof supporter !== 'object') {
-        throw new Error(`Supporters: ${JSON.stringify(supporter)} is not an object.`);
+        throw new Error(
+          `Supporters: ${JSON.stringify(supporter)} is not an object.`
+        );
       }
       if (!(key in supporter)) {
-        throw new Error(`Supporters: ${JSON.stringify(supporter)} doesn't include ${key}.`);
+        throw new Error(
+          `Supporters: ${JSON.stringify(supporter)} doesn't include ${key}.`
+        );
       }
     }
     supportersBySlug.set(supporter.slug, supporter);
   }
 
   // Calculate monthly amount from transactions
-  const transactions = await getAllNodes(transactionsGraphqlQuery, data => data.account.transactions.nodes);
+  const transactions = await getAllNodes(
+    transactionsGraphqlQuery,
+    (data) => data.transactions.nodes
+  );
   for (const transaction of transactions) {
     if (!transaction.amountInHostCurrency) continue;
     const amount = transaction.amountInHostCurrency.value;
     if (!amount || amount <= 0) continue;
-    const date = +new Date(transaction.createdAt);
-    if (date < oneYearAgo) continue;
     const supporter = supportersBySlug.get(transaction.fromAccount.slug);
     if (!supporter) continue;
-    supporter.monthlyDonations += amount * 100 / 12;
+    supporter.monthlyDonations += (amount * 100) / 12;
   }
 
   for (const supporter of supporters) {
     supporter.monthlyDonations = Math.round(supporter.monthlyDonations);
   }
 
   // Write the file
-  return asyncWriteFile(absoluteFilename, JSON.stringify(supporters, null, 2)).then(() =>
-    console.log(`Fetched 1 file: ${filename}`)
-  );
-})().catch(error => {
+  return asyncWriteFile(
+    absoluteFilename,
+    JSON.stringify(supporters, null, 2)
+  ).then(() => console.log(`Fetched 1 file: ${filename}`));
+})().catch((error) => {
   console.error('utilities/fetch-supporters:', error);
   process.exitCode = 1;
 });