diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0381bd149181..e8379e940e58 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v7 - name: Use Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 24 package-manager-cache: false diff --git a/.github/workflows/publish-to-npm.yml b/.github/workflows/publish-to-npm.yml index a2ed49959f18..33b51de4da1b 100644 --- a/.github/workflows/publish-to-npm.yml +++ b/.github/workflows/publish-to-npm.yml @@ -44,7 +44,7 @@ jobs: fetch-depth: 0 - name: Use Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 24 package-manager-cache: false @@ -55,7 +55,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d3b8d6eda6a..446ffa5f849f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,7 +46,7 @@ jobs: uses: actions/checkout@v7 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: ${{ matrix.node-version }} package-manager-cache: false @@ -57,7 +57,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' @@ -105,7 +105,7 @@ jobs: fetch-depth: 0 - name: Use Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 24 package-manager-cache: false @@ -116,7 +116,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' @@ -200,7 +200,7 @@ jobs: fetch-depth: 0 - name: Use Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 24 package-manager-cache: false @@ -214,7 +214,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index 61d211787a5a..039964e5c578 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@v7 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: ${{ matrix.node-version }} package-manager-cache: false @@ -45,7 +45,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' @@ -109,7 +109,7 @@ jobs: uses: actions/checkout@v7 - name: Use Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 24 package-manager-cache: false @@ -120,7 +120,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' @@ -251,7 +251,7 @@ jobs: uses: actions/checkout@v7 - name: Use Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 24 package-manager-cache: false @@ -262,7 +262,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' @@ -298,7 +298,7 @@ jobs: fetch-depth: 0 - name: Use Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 24 package-manager-cache: false @@ -309,7 +309,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 1cccfccbfce2..d0e460cde224 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v7 - name: Use Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 24 package-manager-cache: false @@ -40,7 +40,7 @@ jobs: corepack prepare yarn@stable --activate - name: Activate cache for Node.js 24 - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: cache: 'yarn' @@ -87,6 +87,8 @@ jobs: run: yarn test:e2e env: STORAGE_IMPLEMENTATION: ${{ matrix.storage }} + # Provided explicitly: the CLI may store the token in the OS keyring, where the tests can't read it. + APIFY_TOKEN: ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }} APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GITHUB_TOKEN: ${{ github.token }} diff --git a/packages/core/src/cookie_utils.ts b/packages/core/src/cookie_utils.ts index a97477ce7370..7e8f4e12e531 100644 --- a/packages/core/src/cookie_utils.ts +++ b/packages/core/src/cookie_utils.ts @@ -116,7 +116,8 @@ export function mergeCookies(url: string, sourceCookies: string[]): string { // ignore extra spaces if (!cookieString) continue; - const cookie = Cookie.parse(cookieString)!; + const cookie = Cookie.parse(cookieString); + if (!cookie) throw new CookieParseError(cookieString); const similarKeyCookie = jar.getCookiesSync(url).find((c) => { return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase(); }); diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts index cfba7f7b8044..5696dbc3e1e9 100644 --- a/test/core/crawlers/cheerio_crawler.test.ts +++ b/test/core/crawlers/cheerio_crawler.test.ts @@ -1169,6 +1169,15 @@ describe('CheerioCrawler', () => { ); }); + test('mergeCookies() throws a contextual error for malformed cookie fragments', () => { + expect(() => mergeCookies('https://example.com', ['valid=1; brokenfragment'])).toThrow( + 'Could not parse cookie header string: brokenfragment', + ); + expect(() => mergeCookies('https://example.com', ['sessionid'])).toThrow( + 'Could not parse cookie header string: sessionid', + ); + }); + test('should use sessionId in proxyUrl when the session pool is enabled', async () => { const sourcesNew = [{ url: 'http://example.com/?q=1' }]; const requestListNew = await RequestList.open({ sources: sourcesNew }); diff --git a/test/e2e/adaptive-playwright-robots-file/actor/main.js b/test/e2e/adaptive-playwright-robots-file/actor/main.js index 96a87f86834b..d3355bc1f865 100644 --- a/test/e2e/adaptive-playwright-robots-file/actor/main.js +++ b/test/e2e/adaptive-playwright-robots-file/actor/main.js @@ -9,6 +9,8 @@ await Actor.init({ }); const crawler = new AdaptivePlaywrightCrawler({ + // The store rate-limits the platform's shared egress IP, so crawl through a proxy. + proxyConfiguration: await Actor.createProxyConfiguration(), maxRequestsPerCrawl: 10, respectRobotsTxtFile: true, onSkippedRequest: (args) => crawler.log.warningOnce(`Request ${args.url} was skipped, reason: ${args.reason}`), diff --git a/test/e2e/cheerio-robots-file/actor/main.js b/test/e2e/cheerio-robots-file/actor/main.js index db191d417c90..820f4be4bbf8 100644 --- a/test/e2e/cheerio-robots-file/actor/main.js +++ b/test/e2e/cheerio-robots-file/actor/main.js @@ -9,6 +9,8 @@ await Actor.init({ }); const crawler = new CheerioCrawler({ + // The store rate-limits the platform's shared egress IP, so crawl through a proxy. + proxyConfiguration: await Actor.createProxyConfiguration(), maxRequestsPerCrawl: 10, respectRobotsTxtFile: true, }); diff --git a/test/e2e/playwright-introduction-guide/actor/main.js b/test/e2e/playwright-introduction-guide/actor/main.js index 5d232eed6614..f6cbd281f059 100644 --- a/test/e2e/playwright-introduction-guide/actor/main.js +++ b/test/e2e/playwright-introduction-guide/actor/main.js @@ -89,6 +89,8 @@ router.addDefaultHandler(async ({ request, page, enqueueLinks, log }) => { }); const crawler = new PlaywrightCrawler({ + // The store rate-limits the platform's shared egress IP, so crawl through a proxy. + proxyConfiguration: await Actor.createProxyConfiguration(), maxRequestsPerCrawl: 15, // so the test runs faster // Instead of the long requestHandler with // if clauses we provide a router instance. diff --git a/test/e2e/playwright-robots-file/actor/main.js b/test/e2e/playwright-robots-file/actor/main.js index f38eb3e495c0..c50ba6939395 100644 --- a/test/e2e/playwright-robots-file/actor/main.js +++ b/test/e2e/playwright-robots-file/actor/main.js @@ -9,6 +9,8 @@ await Actor.init({ }); const crawler = new PlaywrightCrawler({ + // The store rate-limits the platform's shared egress IP, so crawl through a proxy. + proxyConfiguration: await Actor.createProxyConfiguration(), maxRequestsPerCrawl: 10, respectRobotsTxtFile: true, }); diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js index 635d6f9113af..c78ef824182b 100644 --- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js +++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js @@ -11,6 +11,8 @@ const mainOptions = { await Actor.main(async () => { const crawler = new PuppeteerCrawler({ + // The store rate-limits the platform's shared egress IP, so crawl through a proxy. + proxyConfiguration: await Actor.createProxyConfiguration(), maxRequestsPerCrawl: 10, preNavigationHooks: [ async ({ page }, goToOptions) => { diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.js index 85e4a355e70d..c9e1b0da758f 100644 --- a/test/e2e/puppeteer-store-pagination/actor/main.js +++ b/test/e2e/puppeteer-store-pagination/actor/main.js @@ -9,6 +9,8 @@ await Actor.init({ }); const crawler = new PuppeteerCrawler({ + // The store rate-limits the platform's shared egress IP, so crawl through a proxy. + proxyConfiguration: await Actor.createProxyConfiguration(), maxRequestsPerCrawl: 10, preNavigationHooks: [ async ({ page }, goToOptions) => { diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs index be30e5480ace..c47615d26be9 100644 --- a/test/e2e/tools.mjs +++ b/test/e2e/tools.mjs @@ -339,6 +339,15 @@ export async function getApifyToken() { } const { token } = await fs.readJSON(authPath); + + // Newer CLI versions keep the token in the OS keyring, leaving auth.json without it. + if (!token) { + throw new Error( + 'Your Apify token is not stored in auth.json (the CLI likely saved it in the OS keyring). ' + + 'Set the "APIFY_TOKEN" environment variable, or re-run "apify login" with "APIFY_DISABLE_KEYRING=1".', + ); + } + return token; }