github · Octomerger · Aug 29, 2022 · Aug 29, 2022 · Aug 29, 2022 · Aug 29, 2022
diff --git a/.github/workflows/link-check-all.yml b/.github/workflows/link-check-all.yml
@@ -53,6 +53,14 @@ jobs:
           # Don't care about CDN caching image URLs
           DISABLE_REWRITE_ASSET_URLS: true
         run: |
+
+          # Note as of Aug 2022, we *don't* check external links
+          # on the pages you touched in the PR. We could enable that
+          # but it has the added risk of false positives blocking CI.
+          # We are using this script for the daily/nightly checker that
+          # checks external links too. Once we're confident it really works
+          # well, we can consider enabling it here on every content PR too.
+
           ./script/rendered-content-link-checker.js \
             --language en \
             --max 100 \

diff --git a/script/rendered-content-link-checker.js b/script/rendered-content-link-checker.js
@@ -12,6 +12,7 @@ import path from 'path'
 import cheerio from 'cheerio'
 import { program, Option, InvalidArgumentError } from 'commander'
 import chalk from 'chalk'
+import got from 'got'
 
 import shortVersions from '../middleware/contextualizers/short-versions.js'
 import contextualize from '../middleware/context.js'
@@ -20,6 +21,7 @@ import getRedirect from '../lib/get-redirect.js'
 import warmServer from '../lib/warm-server.js'
 import renderContent from '../lib/render-content/index.js'
 import { deprecated } from '../lib/enterprise-server-releases.js'
+import excludedLinks from '../lib/excluded-links.js'
 
 const STATIC_PREFIXES = {
   assets: path.resolve('assets'),
@@ -32,6 +34,18 @@ Object.entries(STATIC_PREFIXES).forEach(([key, value]) => {
   }
 })
 
+// Return a function that can as quickly as possible check if a certain
+// href input should be skipped.
+// Do this so we can use a `Set` and a `iterable.some()` for a speedier
+// check.
+function linksToSkipFactory() {
+  const set = new Set(excludedLinks.filter((regexOrURL) => typeof regexOrURL === 'string'))
+  const regexes = excludedLinks.filter((regexOrURL) => regexOrURL instanceof RegExp)
+  return (href) => set.has(href) || regexes.some((regex) => regex.test(href))
+}
+
+const linksToSkip = linksToSkipFactory(excludedLinks)
+
 const CONTENT_ROOT = path.resolve('content')
 
 const deprecatedVersionPrefixesRegex = new RegExp(
@@ -56,6 +70,7 @@ program
   .option('-b, --bail', 'Exit on the first flaw')
   .option('--check-anchors', "Validate links that start with a '#' too")
   .option('--check-images', 'Validate local images too')
+  .option('--check-external-links', 'Check external URLs too')
   .option('-v, --verbose', 'Verbose outputs')
   .option('--debug', "Loud about everything it's doing")
   .option('--random', 'Load pages in a random order (useful for debugging)')
@@ -92,7 +107,7 @@ program
 main(program.opts(), program.args)
 
 async function main(opts, files) {
-  const { random, language, filter, exit, debug, max, verbose, list } = opts
+  const { random, language, filter, exit, debug, max, verbose, list, checkExternalLinks } = opts
 
   // Note! The reason we're using `warmServer()` in this script,
   // even though there's no server involved, is because
@@ -133,6 +148,14 @@ async function main(opts, files) {
   const pages = getPages(pageList, languages, filters, files, max)
   debug && console.timeEnd('getPages')
 
+  if (checkExternalLinks && pages.length >= 100) {
+    console.warn(
+      chalk.yellow(
+        `Warning! Checking external URLs can be time costly. You're testing ${pages.length} pages.`
+      )
+    )
+  }
+
   const processPagesStart = new Date()
   const flawsGroups = await Promise.all(
     pages.map((page) => processPage(page, pageMap, redirects, opts))
@@ -240,38 +263,58 @@ async function processPage(page, pageMap, redirects, opts) {
 }
 
 async function processPermalink(permalink, page, pageMap, redirects, opts) {
-  const { level, checkAnchors, checkImages } = opts
+  const { level, checkAnchors, checkImages, checkExternalLinks } = opts
   const html = await renderInnerHTML(page, permalink)
   const $ = cheerio.load(html)
   const flaws = []
+  const links = []
   $('a[href]').each((i, link) => {
-    const { href } = link.attribs
-
-    // The global cache can't be used for anchor links because they
-    // depend on each page it renders
-    if (!href.startsWith('#')) {
-      if (globalHrefCheckCache.has(href)) {
-        globalCacheHitCount++
-        return globalHrefCheckCache.get(href)
+    links.push(link)
+  })
+  const newFlaws = await Promise.all(
+    links.map(async (link) => {
+      const { href } = link.attribs
+
+      // The global cache can't be used for anchor links because they
+      // depend on each page it renders
+      if (!href.startsWith('#')) {
+        if (globalHrefCheckCache.has(href)) {
+          globalCacheHitCount++
+          return globalHrefCheckCache.get(href)
+        }
+        globalCacheMissCount++
       }
-      globalCacheMissCount++
-    }
-
-    const flaw = checkHrefLink(href, $, redirects, pageMap, checkAnchors)
 
-    // Again if it's *not* an anchor link, we can use the cache.
-    if (!href.startsWith('#')) {
-      globalHrefCheckCache.set(href, flaw)
-    }
+      const flaw = await checkHrefLink(
+        href,
+        $,
+        redirects,
+        pageMap,
+        checkAnchors,
+        checkExternalLinks
+      )
 
-    if (flaw) {
-      if (level === 'critical' && !flaw.CRITICAL) {
-        return
+      if (flaw) {
+        if (level === 'critical' && !flaw.CRITICAL) {
+          return
+        }
+        const text = $(link).text()
+        if (!href.startsWith('#')) {
+          globalHrefCheckCache.set(href, { href, flaw, text })
+        }
+        return { href, flaw, text }
+      } else {
+        if (!href.startsWith('#')) {
+          globalHrefCheckCache.set(href, flaw)
+        }
       }
-      const text = $(link).text()
-      flaws.push({ permalink, page, href, flaw, text })
+    })
+  )
+  for (const flaw of newFlaws) {
+    if (flaw) {
+      flaws.push(Object.assign(flaw, { page, permalink }))
     }
-  })
+  }
 
   if (checkImages) {
     $('img[src]').each((i, img) => {
@@ -353,7 +396,14 @@ const globalImageSrcCheckCache = new Map()
 let globalCacheHitCount = 0
 let globalCacheMissCount = 0
 
-function checkHrefLink(href, $, redirects, pageMap, checkAnchors = false) {
+async function checkHrefLink(
+  href,
+  $,
+  redirects,
+  pageMap,
+  checkAnchors = false,
+  checkExternalLinks = false
+) {
   if (href === '#') {
     if (checkAnchors) {
       return { WARNING: 'Link is just an empty `#`' }
@@ -399,9 +449,80 @@ function checkHrefLink(href, $, redirects, pageMap, checkAnchors = false) {
         return { CRITICAL: 'Broken link' }
       }
     }
+  } else if (checkExternalLinks) {
+    if (!href.startsWith('https://')) {
+      return { WARNING: `Will not check external URLs that are not HTTPS (${href})` }
+    }
+    if (linksToSkip(href)) {
+      return
+    }
+    let failed = false
+
+    try {
+      failed = await checkExternalURL(href)
+    } catch (err) {
+      return { WARNING: `Got error when testing ${href}: ${err.toString()}` }
+    }
+    if (failed) {
+      return { CRITICAL: 'Broken external link ' }
+    }
   }
 }
 
+const externalResponseCache = new Map()
+const externalResponseWaiting = new Set()
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+async function checkExternalURL(url) {
+  if (!url.startsWith('https://')) throw new Error('Invalid URL')
+
+  if (externalResponseCache.has(url)) {
+    const result = externalResponseCache.get(url)
+    return result
+  }
+  if (externalResponseWaiting.has(url)) {
+    // Because this whole script is based on `Promise.all()` you can't
+    // guarantee that you first make the list of external URLs distinct,
+    // so you'll end up with N concurrent threads that both start,
+    // waiting for the same URL to check.
+    // If there's one going on, sleep and retry all over.
+    await sleep(500 + Math.random() * 100)
+    return await checkExternalURL(url)
+  }
+  externalResponseWaiting.add(url)
+
+  // The way `got` does retries:
+  //
+  //   sleep = 1000 * Math.pow(2, retry - 1) + Math.random() * 100
+  //
+  // So, it means:
+  //
+  //   1. ~1000ms
+  //   2. ~2000ms
+  //   3. ~4000ms
+  //
+  // ...if the limit we set is 3.
+  // Our own timeout, in ./middleware/timeout.js defaults to 10 seconds.
+  // So there's no point in trying more attempts than 3 because it would
+  // just timeout on the 10s. (i.e. 1000 + 2000 + 4000 + 8000 > 10,000)
+  const retry = {
+    limit: 3,
+  }
+  const timeout = 2000
+
+  const r = await got(url, {
+    throwHttpErrors: false,
+    retry,
+    timeout,
+  })
+
+  const failed = r.statusCode !== 200
+  externalResponseCache.set(url, failed)
+  externalResponseWaiting.delete(url)
+  return failed
+}
+
 function checkImageSrc(src, $) {
   const pathname = new URL(src, 'http://example.com').pathname
   if (!pathname.startsWith('/')) {

diff --git a/translations/log/cn-resets.csv b/translations/log/cn-resets.csv
@@ -395,16 +395,15 @@ translations/zh-CN/data/reusables/notifications/vulnerable-dependency-notificati
 translations/zh-CN/data/reusables/organizations/team-synchronization.md,broken liquid tags
 translations/zh-CN/data/reusables/package_registry/authenticate-packages.md,broken liquid tags
 translations/zh-CN/data/reusables/package_registry/authenticate-to-container-registry-steps.md,broken liquid tags
-translations/zh-CN/data/reusables/package_registry/authenticate_with_pat_for_container_registry.md,broken liquid tags
 translations/zh-CN/data/reusables/package_registry/docker_registry_deprecation_status.md,Listed in localization-support#489
 translations/zh-CN/data/reusables/package_registry/next-steps-for-packages-enterprise-setup.md,broken liquid tags
 translations/zh-CN/data/reusables/package_registry/packages-cluster-support.md,broken liquid tags
 translations/zh-CN/data/reusables/repositories/deleted_forks_from_private_repositories_warning.md,broken liquid tags
 translations/zh-CN/data/reusables/repositories/enable-security-alerts.md,broken liquid tags
 translations/zh-CN/data/reusables/repositories/select-marketplace-apps.md,broken liquid tags
-translations/zh-CN/data/reusables/saml/saml-session-oauth.md,rendering error
+translations/zh-CN/data/reusables/saml/saml-session-oauth.md,broken liquid tags
 translations/zh-CN/data/reusables/saml/you-must-periodically-authenticate.md,Listed in localization-support#489
-translations/zh-CN/data/reusables/saml/you-must-periodically-authenticate.md,rendering error
+translations/zh-CN/data/reusables/saml/you-must-periodically-authenticate.md,broken liquid tags
 translations/zh-CN/data/reusables/scim/after-you-configure-saml.md,broken liquid tags
 translations/zh-CN/data/reusables/secret-scanning/enterprise-enable-secret-scanning.md,broken liquid tags
 translations/zh-CN/data/reusables/secret-scanning/partner-program-link.md,broken liquid tags

diff --git a/...your-own-runners/configuring-the-self-hosted-runner-application-as-a-service.md b/...your-own-runners/configuring-the-self-hosted-runner-application-as-a-service.md
@@ -18,15 +18,23 @@ shortTitle: 启动时运行运行器应用程序
 
 {% capture service_first_step %}1. 如果自托管的运行器应用程序正在运行，请停止它。{% endcapture %}
 {% capture service_non_windows_intro_shell %}在运行器机器上，在安装了自托管运行器应用程序的目录中打开 shell。 使用以下命令安装和管理自托管的运行器服务。{% endcapture %}
-{% capture service_nonwindows_intro %}将自托管的运行器应用程序配置为服务之前，您必须添加运行器到 {% data variables.product.product_name %}。 更多信息请参阅“[添加自托管的运行器](/github/automating-your-workflow-with-github-actions/adding-self-hosted-runners)”。{% endcapture %}
-{% capture service_win_name %}actions.runner.*{% endcapture %}
 
+{% capture service_nonwindows_intro %}
+
+{% note %}
+
+**Note:** You must add a runner to {% data variables.product.product_name %} before you can configure the self-hosted runner application as a service. 更多信息请参阅“[添加自托管的运行器](/github/automating-your-workflow-with-github-actions/adding-self-hosted-runners)”。
+
+{% endnote %}
+{% endcapture %}
+
+{% capture service_win_name %}actions.runner.*{% endcapture %}
 
 {% linux %}
 
 {{ service_nonwindows_intro }}
 
-对于使用 `systemd` 的 Linux 系统，您可以使用随自托管运行器应用程序分发的 `svc.h` 脚本来安装和管理应用程序即服务。
+For Linux systems that use `systemd`, you can use the `svc.sh` script that is created after successfully adding the runner to install and manage using the application as a service.
 
 {{ service_non_windows_intro_shell }}
 

diff --git a/...lations/zh-CN/content/actions/publishing-packages/publishing-nodejs-packages.md b/...lations/zh-CN/content/actions/publishing-packages/publishing-nodejs-packages.md
@@ -98,9 +98,9 @@ always-auth=true
 
 ### 配置目标仓库
 
-如果您没有在 *package.json* 文件中提供 `repository` 键，则 {% data variables.product.prodname_registry %} 将包发布到您在 *package.json* 文件的 `name` 字段中指定的 {% data variables.product.prodname_dotcom %} 仓库。 例如，名为 `@my-org/test` 的包将被发布到 `my-org/test` {% data variables.product.prodname_dotcom %} 仓库。
+Linking your package to {% data variables.product.prodname_registry %} using the `repository` key is optional. If you choose not to provide the `repository` key in your *package.json* file, then {% data variables.product.prodname_registry %} publishes a package in the {% data variables.product.prodname_dotcom %} repository you specify in the `name` field of the *package.json* file. 例如，名为 `@my-org/test` 的包将被发布到 `my-org/test` {% data variables.product.prodname_dotcom %} 仓库。 If the `url` specified in the `repository` key is invalid, your package may still be published however it won't be linked to the repository source as intended.
 
-但是，如果您提供了 `repository` 键，则该键中的仓库将被用作 {% data variables.product.prodname_registry %} 的目标 npm 注册表。 例如，发布以下 *package.json* 将导致名为 `my-amazing-package` 的包被发布到 `octocat/my-other-repo` {% data variables.product.prodname_dotcom %} 仓库。
+If you do provide the `repository` key in your *package.json* file, then the repository in that key is used as the destination npm registry for {% data variables.product.prodname_registry %}. 例如，发布以下 *package.json* 将导致名为 `my-amazing-package` 的包被发布到 `octocat/my-other-repo` {% data variables.product.prodname_dotcom %} 仓库。 Once published, only the repository source is updated, and the package doesn't inherit any permissions from the destination repository.
 
 ```json
 {

diff --git a/...-your-enterprise/enforcing-policies-for-security-settings-in-your-enterprise.md b/...-your-enterprise/enforcing-policies-for-security-settings-in-your-enterprise.md
@@ -79,6 +79,8 @@ shortTitle: 安全设置策略
 
 {% data reusables.identity-and-permissions.about-adding-ip-allow-list-entries %}
 
+{% data reusables.identity-and-permissions.ipv6-allow-lists %}
+
 {% data reusables.enterprise-accounts.access-enterprise %}
 {% data reusables.enterprise-accounts.settings-tab %}
 {% data reusables.enterprise-accounts.security-tab %}

diff --git a/...code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning.md b/...code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning.md
@@ -45,9 +45,15 @@ If you upload a second SARIF file for a commit with the same category and from t
 
 If you're new to SARIF and want to learn more, see Microsoft's [`SARIF tutorials`](https://github.com/microsoft/sarif-tutorials) repository.
 
-## Preventing duplicate alerts using fingerprints
+## Providing data to track {% data variables.product.prodname_code_scanning %} alerts across runs
 
-Each time the results of a new code scan are uploaded, the results are processed and alerts are added to the repository. To prevent duplicate alerts for the same problem, {% data variables.product.prodname_code_scanning %} uses fingerprints to match results across various runs so they only appear once in the latest run for the selected branch. This makes it possible to match alerts to the right line of code when files are edited.
+Each time the results of a new code scan are uploaded, the results are processed and alerts are added to the repository. To prevent duplicate alerts for the same problem, {% data variables.product.prodname_code_scanning %} uses fingerprints to match results across various runs so they only appear once in the latest run for the selected branch. This makes it possible to match alerts to the correct line of code when files are edited. The `ruleID` for a result has to be the same across analysis.
+
+### Reporting consistent filepaths
+
+The filepath has to be consistent across the runs to enable a computation of a stable fingerprint. If the filepaths differ for the same result, each time there is a new analysis a new alert will be created, and the old one will be closed. This will cause having multiple alerts for the same result.
+
+### Including data for fingerprint generation
 
 {% data variables.product.prodname_dotcom %} uses the `partialFingerprints` property in the OASIS standard to detect when two results are logically identical. For more information, see the "[partialFingerprints property](https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Toc16012611)" entry in the OASIS documentation.
 
@@ -138,7 +144,7 @@ Each `result` object contains details for one alert in the codebase. Within the
 | `level`| **Optional.** The severity of the result. This level overrides the default severity defined by the rule. {% data variables.product.prodname_code_scanning_capc %} uses the level to filter results by severity on {% data variables.product.prodname_dotcom %}.
 | `message.text`| **Required.** A message that describes the result. {% data variables.product.prodname_code_scanning_capc %} displays the message text as the title of the result. Only the first sentence of the message will be displayed when visible space is limited.
 | `locations[]`| **Required.** The set of locations where the result was detected up to a maximum of 10. Only one location should be included unless the problem can only be corrected by making a change at every specified location. **Note:** At least one location is required for {% data variables.product.prodname_code_scanning %} to display a result. {% data variables.product.prodname_code_scanning_capc %} will use this property to decide which file to annotate with the result. Only the first value of this array is used. All other values are ignored.
-| `partialFingerprints`| **Required.** A set of strings used to track the unique identity of the result. {% data variables.product.prodname_code_scanning_capc %} uses `partialFingerprints` to accurately identify which results are the same across commits and branches. {% data variables.product.prodname_code_scanning_capc %} will attempt to use `partialFingerprints` if they exist. If you are uploading third-party SARIF files with the `upload-action`, the action will create `partialFingerprints` for you when they are not included in the SARIF file. For more information, see "[Preventing duplicate alerts using fingerprints](#preventing-duplicate-alerts-using-fingerprints)."  **Note:** {% data variables.product.prodname_code_scanning_capc %} only uses the `primaryLocationLineHash`.
+| `partialFingerprints`| **Required.** A set of strings used to track the unique identity of the result. {% data variables.product.prodname_code_scanning_capc %} uses `partialFingerprints` to accurately identify which results are the same across commits and branches. {% data variables.product.prodname_code_scanning_capc %} will attempt to use `partialFingerprints` if they exist. If you are uploading third-party SARIF files with the `upload-action`, the action will create `partialFingerprints` for you when they are not included in the SARIF file. For more information, see "[Providing data to track code scanning alerts across runs](#providing-data-to-track-code-scanning-alerts-across-runs)."  **Note:** {% data variables.product.prodname_code_scanning_capc %} only uses the `primaryLocationLineHash`.
 | `codeFlows[].threadFlows[].locations[]`| **Optional.** An array of `location` objects for a `threadFlow` object, which describes the progress of a program through a thread of execution. A `codeFlow` object describes a pattern of code execution used to detect a result. If code flows are provided, {% data variables.product.prodname_code_scanning %} will expand code flows on {% data variables.product.prodname_dotcom %} for the relevant result. For more information, see the [`location` object](#location-object).
 | `relatedLocations[]`| A set of locations relevant to this result. {% data variables.product.prodname_code_scanning_capc %} will link to related locations when they are embedded in the result message. For more information, see the [`location` object](#location-object).