dd-trace - npm Package Compare versions

Comparing version 6.0.0-pre-09455e6 to 6.0.0-pre-1106f0c

package.json

 {
   "name": "dd-trace",
-  "version": "6.0.0-pre-09455e6",
+  "version": "6.0.0-pre-1106f0c",
   "description": "Datadog APM tracing client for JavaScript",
@@ -72,7 +72,7 @@ "main": "index.js",
   "dependencies": {
-    "@datadog/native-appsec": "7.1.0",
+    "@datadog/native-appsec": "7.1.1",
     "@datadog/native-iast-rewriter": "2.3.0",
     "@datadog/native-iast-taint-tracking": "1.7.0",
     "@datadog/native-metrics": "^2.0.0",
-    "@datadog/pprof": "5.1.0",
+    "@datadog/pprof": "5.2.0",
     "@datadog/sketches-js": "^2.1.0",
@@ -99,3 +99,3 @@ "@opentelemetry/api": "^1.0.0",
     "path-to-regexp": "^0.1.2",
-    "pprof-format": "^2.0.7",
+    "pprof-format": "^2.1.0",
     "protobufjs": "^7.2.5",
@@ -102,0 +102,0 @@ "retry": "^0.13.1",

packages/datadog-instrumentations/src/jest.js

@@ -14,3 +14,4 @@ 'use strict'
   addEfdStringToTestName,
-  removeEfdStringFromTestName
+  removeEfdStringFromTestName,
+  getIsFaultyEarlyFlakeDetection
 } = require('../../dd-trace/src/plugins/util/test')
@@ -48,2 +49,5 @@ const {
 
+// Message sent by jest's main process to workers to run a test suite (=test file)
+// https://github.com/jestjs/jest/blob/1d682f21c7a35da4d3ab3a1436a357b980ebd0fa/packages/jest-worker/src/types.ts#L37
+const CHILD_MESSAGE_CALL = 1
 // Maximum time we'll wait for the tracer to flush
@@ -53,3 +57,3 @@ const FLUSH_TIMEOUT = 10000
 let skippableSuites = []
-let knownTests = []
+let knownTests = {}
 let isCodeCoverageEnabled = false
@@ -64,2 +68,4 @@ let isSuitesSkippingEnabled = false
 let earlyFlakeDetectionNumRetries = 0
+let earlyFlakeDetectionFaultyThreshold = 30
+let isEarlyFlakeDetectionFaulty = false
 let hasFilteredSkippableSuites = false
@@ -80,2 +86,3 @@
 const retriedTestsToNumAttempts = new Map()
+const newTestsTestStatuses = new Map()
 
@@ -109,2 +116,9 @@ // based on https://github.com/facebook/jest/blob/main/packages/jest-circus/src/formatNodeAssertErrors.ts#L41
 
+function getEfdStats (testStatuses) {
+  return testStatuses.reduce((acc, testStatus) => {
+    acc[testStatus]++
+    return acc
+  }, { pass: 0, fail: 0 })
+}
+
 function getWrappedEnvironment (BaseEnvironment, jestVersion) {
@@ -119,2 +133,3 @@ return class DatadogEnvironment extends BaseEnvironment {
       this.global._ddtrace = global._ddtrace
+      this.hasSnapshotTests = undefined
 
@@ -133,5 +148,8 @@ this.displayName = config.projectConfig?.displayName?.name
       if (this.isEarlyFlakeDetectionEnabled) {
+        const hasKnownTests = !!knownTests.jest
         earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries
         try {
-          this.knownTestsForThisSuite = this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests)
+          this.knownTestsForThisSuite = hasKnownTests
+            ? (knownTests.jest[this.testSuite] || [])
+            : this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests)
         } catch (e) {
@@ -144,2 +162,17 @@ // If there has been an error parsing the tests, we'll disable Early Flake Deteciton
 
+    getHasSnapshotTests () {
+      if (this.hasSnapshotTests !== undefined) {
+        return this.hasSnapshotTests
+      }
+      let hasSnapshotTests = true
+      try {
+        const { _snapshotData } = this.context.expect.getState().snapshotState
+        hasSnapshotTests = Object.keys(_snapshotData).length > 0
+      } catch (e) {
+        // if we can't be sure, we'll err on the side of caution and assume it has snapshots
+      }
+      this.hasSnapshotTests = hasSnapshotTests
+      return hasSnapshotTests
+    }
+
     // Function that receives a list of known tests for a test service and
@@ -157,3 +190,3 @@ // returns the ones that belong to the current suite
       }
-      return knownTestsForSuite.jest?.[this.testSuite] || []
+      return knownTestsForSuite
     }
@@ -230,2 +263,9 @@
             retriedTestsToNumAttempts.set(testName, 0)
+            // Retrying snapshots has proven to be problematic, so we'll skip them for now
+            // We'll still detect new tests, but we won't retry them.
+            // TODO: do not bail out of EFD with the whole test suite
+            if (this.getHasSnapshotTests()) {
+              log.warn('Early flake detection is disabled for suites with snapshots')
+              return
+            }
             for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) {
@@ -256,2 +296,15 @@ if (this.global.test) {
           event.test.fn = originalTestFns.get(event.test)
+          // We'll store the test statuses of the retries
+          if (this.isEarlyFlakeDetectionEnabled) {
+            const testName = getJestTestName(event.test)
+            const originalTestName = removeEfdStringFromTestName(testName)
+            const isNewTest = retriedTestsToNumAttempts.has(originalTestName)
+            if (isNewTest) {
+              if (newTestsTestStatuses.has(originalTestName)) {
+                newTestsTestStatuses.get(originalTestName).push(status)
+              } else {
+                newTestsTestStatuses.set(originalTestName, [status])
+              }
+            }
+          }
         })
@@ -398,2 +451,3 @@ }
         earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
+        earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold
       }
@@ -513,2 +567,3 @@ } catch (err) {
         isEarlyFlakeDetectionEnabled,
+        isEarlyFlakeDetectionFaulty,
         onDone
@@ -525,2 +580,24 @@ })
 
+    /**
+     * If Early Flake Detection (EFD) is enabled the logic is as follows:
+     * - If all attempts for a test are failing, the test has failed and we will let the test process fail.
+     * - If just a single attempt passes, we will prevent the test process from failing.
+     * The rationale behind is the following: you may still be able to block your CI pipeline by gating
+     * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too.
+     */
+
+    if (isEarlyFlakeDetectionEnabled) {
+      let numFailedTestsToIgnore = 0
+      for (const testStatuses of newTestsTestStatuses.values()) {
+        const { pass, fail } = getEfdStats(testStatuses)
+        if (pass > 0) { // as long as one passes, we'll consider the test passed
+          numFailedTestsToIgnore += fail
+        }
+      }
+      // If every test that failed was an EFD retry, we'll consider the suite passed
+      if (numFailedTestsToIgnore !== 0 && result.results.numFailedTests === numFailedTestsToIgnore) {
+        result.results.success = true
+      }
+    }
+
     return result
@@ -637,3 +714,2 @@ })
     config.testEnvironmentOptions._ddTestCodeCoverageEnabled = isCodeCoverageEnabled
-    config.testEnvironmentOptions._ddKnownTests = knownTests
   })
@@ -731,9 +807,22 @@
   shimmer.wrap(SearchSource.prototype, 'getTestPaths', getTestPaths => async function () {
-    if (!isSuitesSkippingEnabled || !skippableSuites.length) {
-      return getTestPaths.apply(this, arguments)
+    const testPaths = await getTestPaths.apply(this, arguments)
+    const [{ rootDir, shard }] = arguments
+
+    if (isEarlyFlakeDetectionEnabled) {
+      const projectSuites = testPaths.tests.map(test => getTestSuitePath(test.path, test.context.config.rootDir))
+      const isFaulty =
+        getIsFaultyEarlyFlakeDetection(projectSuites, knownTests.jest || {}, earlyFlakeDetectionFaultyThreshold)
+      if (isFaulty) {
+        log.error('Early flake detection is disabled because the number of new suites is too high.')
+        isEarlyFlakeDetectionEnabled = false
+        const testEnvironmentOptions = testPaths.tests[0]?.context?.config?.testEnvironmentOptions
+        // Project config is shared among all tests, so we can modify it here
+        if (testEnvironmentOptions) {
+          testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled = false
+        }
+        isEarlyFlakeDetectionFaulty = true
+      }
     }
 
-    const [{ rootDir, shard }] = arguments
-
-    if (shard?.shardCount > 1) {
+    if (shard?.shardCount > 1 || !isSuitesSkippingEnabled || !skippableSuites.length) {
       // If the user is using jest sharding, we want to apply the filtering of tests in the shard process.
@@ -746,6 +835,4 @@ // The reason for this is the following:
       // causing the shards to potentially run the same suite.
-      return getTestPaths.apply(this, arguments)
+      return testPaths
     }
-
-    const testPaths = await getTestPaths.apply(this, arguments)
     const { tests } = testPaths
@@ -816,2 +903,34 @@
   const ChildProcessWorker = childProcessWorker.default
+  shimmer.wrap(ChildProcessWorker.prototype, 'send', send => function (request) {
+    if (!isEarlyFlakeDetectionEnabled) {
+      return send.apply(this, arguments)
+    }
+    const [type] = request
+    // eslint-disable-next-line
+    // https://github.com/jestjs/jest/blob/1d682f21c7a35da4d3ab3a1436a357b980ebd0fa/packages/jest-worker/src/workers/ChildProcessWorker.ts#L424
+    if (type === CHILD_MESSAGE_CALL) {
+      // This is the message that the main process sends to the worker to run a test suite (=test file).
+      // In here we modify the config.testEnvironmentOptions to include the known tests for the suite.
+      // This way the suite only knows about the tests that are part of it.
+      const args = request[request.length - 1]
+      if (args.length > 1) {
+        return send.apply(this, arguments)
+      }
+      if (!args[0]?.config) {
+        return send.apply(this, arguments)
+      }
+      const [{ globalConfig, config, path: testSuiteAbsolutePath }] = args
+      const testSuite = getTestSuitePath(testSuiteAbsolutePath, globalConfig.rootDir || process.cwd())
+      const suiteKnownTests = knownTests.jest?.[testSuite] || []
+      args[0].config = {
+        ...config,
+        testEnvironmentOptions: {
+          ...config.testEnvironmentOptions,
+          _ddKnownTests: suiteKnownTests
+        }
+      }
+    }
+
+    return send.apply(this, arguments)
+  })
   shimmer.wrap(ChildProcessWorker.prototype, '_onMessage', _onMessage => function () {
@@ -818,0 +937,0 @@ const [code, data] = arguments[0]

packages/datadog-instrumentations/src/kafkajs.js

@@ -71,3 +71,6 @@ 'use strict'
           result.then(
-            innerAsyncResource.bind(() => producerFinishCh.publish(undefined)),
+            innerAsyncResource.bind(res => {
+              producerFinishCh.publish(undefined)
+              producerCommitCh.publish(res)
+            }),
             innerAsyncResource.bind(err => {
@@ -81,8 +84,2 @@ if (err) {
 
-          result.then(res => {
-            if (producerCommitCh.hasSubscribers) {
-              producerCommitCh.publish(res)
-            }
-          })
-
           return result
@@ -89,0 +86,0 @@ } catch (e) {

packages/datadog-instrumentations/src/mongoose.js

@@ -24,3 +24,4 @@ 'use strict'
 }, mongoose => {
-  if (mongoose.Promise !== global.Promise) {
+  // As of Mongoose 7, custom promise libraries are no longer supported and mongoose.Promise may be undefined
+  if (mongoose.Promise && mongoose.Promise !== global.Promise) {
     shimmer.wrap(mongoose.Promise.prototype, 'then', wrapThen)
@@ -27,0 +28,0 @@ }

packages/datadog-instrumentations/src/oracledb.js

@@ -24,3 +24,3 @@ 'use strict'
 
-addHook({ name: 'oracledb', versions: ['5'] }, oracledb => {
+addHook({ name: 'oracledb', versions: ['>=5'] }, oracledb => {
   shimmer.wrap(oracledb.Connection.prototype, 'execute', execute => {
@@ -27,0 +27,0 @@ return function wrappedExecute (dbQuery, ...args) {

packages/datadog-plugin-cucumber/src/index.js

@@ -19,3 +19,3 @@ 'use strict'
   TEST_SOURCE_FILE,
-  TEST_EARLY_FLAKE_IS_ENABLED,
+  TEST_EARLY_FLAKE_ENABLED,
   TEST_IS_NEW,
@@ -72,3 +72,3 @@ TEST_IS_RETRY
       if (isEarlyFlakeDetectionEnabled) {
-        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_IS_ENABLED, 'true')
+        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true')
       }
@@ -75,0 +75,0 @@

packages/datadog-plugin-cypress/src/cypress-plugin.js

@@ -31,3 +31,3 @@ const {
   TEST_IS_RETRY,
-  TEST_EARLY_FLAKE_IS_ENABLED
+  TEST_EARLY_FLAKE_ENABLED
 } = require('../../dd-trace/src/plugins/util/test')
@@ -368,3 +368,3 @@ const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util')
     if (this.isEarlyFlakeDetectionEnabled) {
-      testSessionSpanMetadata[TEST_EARLY_FLAKE_IS_ENABLED] = 'true'
+      testSessionSpanMetadata[TEST_EARLY_FLAKE_ENABLED] = 'true'
     }
@@ -371,0 +371,0 @@

packages/datadog-plugin-cypress/src/support.js

@@ -7,2 +7,12 @@ /* eslint-disable */
 
+// If the test is using multi domain with cy.origin, trying to access
+// window properties will result in a cross origin error.
+function safeGetRum (window) {
+  try {
+    return window.DD_RUM
+  } catch (e) {
+    return null
+  }
+}
+
 function isNewTest (test) {
@@ -66,3 +76,3 @@ return !knownTestsForSuite.includes(test.fullTitle())
   cy.window().then(win => {
-    if (win.DD_RUM) {
+    if (safeGetRum(win)) {
       win.dispatchEvent(new Event('beforeunload'))
@@ -89,7 +99,13 @@ }
 
-    if (win.DD_RUM) {
+    if (safeGetRum(win)) {
       testInfo.isRUMActive = true
     }
-    cy.task('dd:afterEach', { test: testInfo, coverage: win.__coverage__ })
+    let coverage
+    try {
+      coverage = win.__coverage__
+    } catch (e) {
+      // ignore error and continue
+    }
+    cy.task('dd:afterEach', { test: testInfo, coverage })
   })
 })

packages/datadog-plugin-jest/src/index.js

@@ -21,3 +21,4 @@ const CiPlugin = require('../../dd-trace/src/plugins/ci_plugin')
   TEST_IS_RETRY,
-  TEST_EARLY_FLAKE_IS_ENABLED,
+  TEST_EARLY_FLAKE_ENABLED,
+  TEST_EARLY_FLAKE_ABORT_REASON,
   JEST_DISPLAY_NAME
@@ -93,2 +94,3 @@ } = require('../../dd-trace/src/plugins/util/test')
       isEarlyFlakeDetectionEnabled,
+      isEarlyFlakeDetectionFaulty,
       onDone
@@ -120,4 +122,7 @@ }) => {
       if (isEarlyFlakeDetectionEnabled) {
-        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_IS_ENABLED, 'true')
+        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true')
       }
+      if (isEarlyFlakeDetectionFaulty) {
+        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')
+      }
 
@@ -124,0 +129,0 @@ this.testModuleSpan.finish()

packages/datadog-plugin-mocha/src/index.js

@@ -23,3 +23,3 @@ 'use strict'
   TEST_IS_RETRY,
-  TEST_EARLY_FLAKE_IS_ENABLED
+  TEST_EARLY_FLAKE_ENABLED
 } = require('../../dd-trace/src/plugins/util/test')
@@ -227,3 +227,3 @@ const { COMPONENT } = require('../../dd-trace/src/constants')
         if (isEarlyFlakeDetectionEnabled) {
-          this.testSessionSpan.setTag(TEST_EARLY_FLAKE_IS_ENABLED, 'true')
+          this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true')
         }
@@ -230,0 +230,0 @@

packages/datadog-plugin-openai/src/services.js

 'use strict'
 
-const { DogStatsDClient, NoopDogStatsDClient } = require('../../dd-trace/src/dogstatsd')
+const { DogStatsDClient } = require('../../dd-trace/src/dogstatsd')
+const NoopDogStatsDClient = require('../../dd-trace/src/noop/dogstatsd')
 const { ExternalLogger, NoopExternalLogger } = require('../../dd-trace/src/external-logger/src')
@@ -5,0 +6,0 @@

packages/datadog-plugin-playwright/src/index.js

@@ -17,3 +17,3 @@ 'use strict'
   TEST_IS_RETRY,
-  TEST_EARLY_FLAKE_IS_ENABLED
+  TEST_EARLY_FLAKE_ENABLED
 } = require('../../dd-trace/src/plugins/util/test')
@@ -45,3 +45,3 @@ const { RESOURCE_NAME } = require('../../../ext/tags')
       if (isEarlyFlakeDetectionEnabled) {
-        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_IS_ENABLED, 'true')
+        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true')
       }
@@ -48,0 +48,0 @@

packages/dd-trace/src/appsec/iast/analyzers/weak-hash-analyzer.js

@@ -23,7 +23,11 @@ 'use strict'
   'sqreen/lib/package-reader/index.js',
-  'ws/lib/websocket-server.js'
+  'ws/lib/websocket-server.js',
+  'google-gax/build/src/grpc.js',
+  'cookie-signature/index.js'
 )
 
 const EXCLUDED_PATHS_FROM_STACK = [
-  path.join('node_modules', 'object-hash', path.sep)
+  path.join('node_modules', 'object-hash', path.sep),
+  path.join('node_modules', 'aws-sdk', 'lib', 'util.js'),
+  path.join('node_modules', 'keygrip', path.sep)
 ]
@@ -30,0 +34,0 @@ class WeakHashAnalyzer extends Analyzer {

packages/dd-trace/src/appsec/reporter.js

@@ -11,3 +11,4 @@ 'use strict'
   incrementWafUpdatesMetric,
-  incrementWafRequestsMetric
+  incrementWafRequestsMetric,
+  getRequestMetrics
 } = require('./telemetry')
@@ -96,3 +97,2 @@ const zlib = require('zlib')
 function reportMetrics (metrics) {
-  // TODO: metrics should be incremental, there already is an RFC to report metrics
   const store = storage.getStore()
@@ -102,10 +102,2 @@ const rootSpan = store?.req && web.root(store.req)
 
-  if (metrics.duration) {
-    rootSpan.setTag('_dd.appsec.waf.duration', metrics.duration)
-  }
-
-  if (metrics.durationExt) {
-    rootSpan.setTag('_dd.appsec.waf.duration_ext', metrics.durationExt)
-  }
-
   if (metrics.rulesVersion) {
@@ -185,2 +177,11 @@ rootSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion)
 
+  const metrics = getRequestMetrics(req)
+  if (metrics?.duration) {
+    rootSpan.setTag('_dd.appsec.waf.duration', metrics.duration)
+  }
+
+  if (metrics?.durationExt) {
+    rootSpan.setTag('_dd.appsec.waf.duration_ext', metrics.durationExt)
+  }
+
   incrementWafRequestsMetric(req)
@@ -187,0 +188,0 @@

packages/dd-trace/src/appsec/telemetry.js

@@ -8,2 +8,3 @@ 'use strict'
 const DD_TELEMETRY_WAF_RESULT_TAGS = Symbol('_dd.appsec.telemetry.waf.result.tags')
+const DD_TELEMETRY_REQUEST_METRICS = Symbol('_dd.appsec.telemetry.request.metrics')
 
@@ -30,6 +31,15 @@ const tags = {
 
+function newStore () {
+  return {
+    [DD_TELEMETRY_REQUEST_METRICS]: {
+      duration: 0,
+      durationExt: 0
+    }
+  }
+}
+
 function getStore (req) {
   let store = metricsStoreMap.get(req)
   if (!store) {
-    store = {}
+    store = newStore()
     metricsStoreMap.set(req, store)
@@ -56,5 +66,3 @@ }
 
-function getOrCreateMetricTags (req, versionsTags) {
-  const store = getStore(req)
-
+function getOrCreateMetricTags (store, versionsTags) {
   let metricTags = store[DD_TELEMETRY_WAF_RESULT_TAGS]
@@ -75,4 +83,11 @@ if (!metricTags) {
 function updateWafRequestsMetricTags (metrics, req) {
-  if (!req || !enabled) return
+  if (!req) return
 
+  const store = getStore(req)
+
+  // it does not depend on whether telemetry is enabled or not
+  addRequestMetrics(store, metrics)
+
+  if (!enabled) return
+
   const versionsTags = getVersionsTags(metrics.wafVersion, metrics.rulesVersion)
@@ -82,3 +97,3 @@
 
-  const metricTags = getOrCreateMetricTags(req, versionsTags)
+  const metricTags = getOrCreateMetricTags(store, versionsTags)
 
@@ -129,2 +144,14 @@ const { blockTriggered, ruleTriggered, wafTimeout } = metrics
 
+function addRequestMetrics (store, { duration, durationExt }) {
+  store[DD_TELEMETRY_REQUEST_METRICS].duration += duration || 0
+  store[DD_TELEMETRY_REQUEST_METRICS].durationExt += durationExt || 0
+}
+
+function getRequestMetrics (req) {
+  if (req) {
+    const store = getStore(req)
+    return store?.[DD_TELEMETRY_REQUEST_METRICS]
+  }
+}
+
 module.exports = {
@@ -137,3 +164,5 @@ enable,
   incrementWafUpdatesMetric,
-  incrementWafRequestsMetric
+  incrementWafRequestsMetric,
+
+  getRequestMetrics
 }

packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -192,3 +192,4 @@ 'use strict'
       isEarlyFlakeDetectionEnabled,
-      earlyFlakeDetectionNumRetries
+      earlyFlakeDetectionNumRetries,
+      earlyFlakeDetectionFaultyThreshold
     } = remoteConfiguration
@@ -201,3 +202,4 @@ return {
       isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled,
-      earlyFlakeDetectionNumRetries
+      earlyFlakeDetectionNumRetries,
+      earlyFlakeDetectionFaultyThreshold
     }
@@ -204,0 +206,0 @@ }

packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -117,2 +117,6 @@ const fs = require('fs')
 
+    if (commitsToUpload === null) {
+      return callback(new Error('git rev-list failed'))
+    }
+
     callback(null, commitsToUpload)
@@ -256,5 +260,4 @@ })
 
-  const latestCommits = getLatestCommits()
+  let latestCommits = getLatestCommits()
   log.debug(`There were ${latestCommits.length} commits since last month.`)
-  const [headCommit] = latestCommits
 
@@ -273,2 +276,3 @@ const getOnFinishGetCommitsToUpload = (hasCheckedShallow) => (err, commitsToUpload) => {
     if (hasCheckedShallow || !isShallowRepository()) {
+      const [headCommit] = latestCommits
       return generateAndUploadPackFiles({
@@ -286,2 +290,5 @@ url,
     unshallowRepository()
+
+    // The latest commits change after unshallowing
+    latestCommits = getLatestCommits()
     getCommitsToUpload({
@@ -288,0 +295,0 @@ url,

packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js

@@ -15,2 +15,3 @@ const request = require('../../exporters/common/request')
 const DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES = 2
+const DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD = 30
 
@@ -108,3 +109,5 @@ function getLibraryConfiguration ({
           earlyFlakeDetectionNumRetries:
-            earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES
+            earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES,
+          earlyFlakeDetectionFaultyThreshold:
+            earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD
         }
@@ -111,0 +114,0 @@

packages/dd-trace/src/config.js

@@ -30,2 +30,3 @@ 'use strict'
 const defaultWafObfuscatorValueRegex = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}'
+const runtimeId = uuid()
 
@@ -295,3 +296,3 @@ function maybeFile (filepath) {
       version: this.version,
-      'runtime-id': uuid()
+      'runtime-id': runtimeId
     })
@@ -828,2 +829,3 @@
     tagger.add(tags, options.tracing_tags)
+    if (Object.keys(tags).length) tags['runtime-id'] = runtimeId
 
@@ -830,0 +832,0 @@ this._setUnit(opts, 'sampleRate', options.tracing_sampling_rate)

packages/dd-trace/src/dogstatsd.js

@@ -15,2 +15,3 @@ 'use strict'
 const TYPE_DISTRIBUTION = 'd'
+const TYPE_HISTOGRAM = 'h'
 
@@ -50,2 +51,6 @@ class DogStatsDClient {
 
+  histogram (stat, value, tags) {
+    this._add(stat, value, TYPE_HISTOGRAM, tags)
+  }
+
   flush () {
@@ -185,12 +190,2 @@ const queue = this._enqueue()
 
-class NoopDogStatsDClient {
-  gauge () { }
-
-  increment () { }
-
-  distribution () { }
-
-  flush () { }
-}
-
 // This is a simplified user-facing proxy to the underlying DogStatsDClient instance
@@ -235,2 +230,10 @@ class CustomMetrics {
 
+  histogram (stat, value, tags) {
+    return this.dogstatsd.histogram(
+      stat,
+      value,
+      CustomMetrics.tagTranslator(tags)
+    )
+  }
+
   flush () {
@@ -259,4 +262,3 @@ return this.dogstatsd.flush()
   DogStatsDClient,
-  NoopDogStatsDClient,
   CustomMetrics
 }

packages/dd-trace/src/noop/proxy.js

@@ -5,5 +5,7 @@ 'use strict'
 const NoopAppsecSdk = require('../appsec/sdk/noop')
+const NoopDogStatsDClient = require('./dogstatsd')
 
 const noop = new NoopTracer()
 const noopAppsec = new NoopAppsecSdk()
+const noopDogStatsDClient = new NoopDogStatsDClient()
 
@@ -14,2 +16,3 @@ class Tracer {
     this.appsec = noopAppsec
+    this.dogstatsd = noopDogStatsDClient
   }
@@ -16,0 +19,0 @@

packages/dd-trace/src/opentracing/span.js

@@ -36,2 +36,3 @@ 'use strict'
 
+const startCh = channel('dd-trace:span:start')
 const finishCh = channel('dd-trace:span:finish')
@@ -100,2 +101,3 @@
     spanleak.addSpan(this, operationName)
+    startCh.publish(this)
   }
@@ -102,0 +104,0 @@

packages/dd-trace/src/plugins/util/git.js

@@ -31,3 +31,3 @@ const cp = require('child_process')
 
-const GIT_REV_LIST_MAX_BUFFER = 8 * 1024 * 1024 // 8MB
+const GIT_REV_LIST_MAX_BUFFER = 12 * 1024 * 1024 // 12MB
 
@@ -57,7 +57,11 @@ function sanitizedExec (
     return result
-  } catch (e) {
+  } catch (err) {
     if (errorMetric) {
-      incrementCountMetric(errorMetric.name, { ...errorMetric.tags, exitCode: e.status })
+      incrementCountMetric(errorMetric.name, {
+        ...errorMetric.tags,
+        errorType: err.code,
+        exitCode: err.status || err.errno
+      })
     }
-    log.error(e)
+    log.error(err)
     return ''
@@ -134,3 +138,6 @@ } finally {
     log.error(err)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
+    )
     const upstreamRemote = sanitizedExec('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}'])
@@ -145,3 +152,6 @@ try {
       log.error(err)
-      incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status })
+      incrementCountMetric(
+        TELEMETRY_GIT_COMMAND_ERRORS,
+        { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
+      )
       // We use sanitizedExec here because if this last option fails, we'll give up.
@@ -182,3 +192,6 @@ sanitizedExec(
     log.error(`Get latest commits failed: ${err.message}`)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_local_commits', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'get_local_commits', errorType: err.status }
+    )
     return []
@@ -189,3 +202,3 @@ }
 function getCommitsRevList (commitsToExclude, commitsToInclude) {
-  let result = []
+  let result = null
 
@@ -214,3 +227,6 @@ const commitsToExcludeString = commitsToExclude.map(commit => `^${commit}`)
     log.error(`Get commits to upload failed: ${err.message}`)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_objects', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'get_objects', errorType: err.code, exitCode: err.status || err.errno } // err.status might be null
+    )
   }
@@ -255,3 +271,6 @@ distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_objects' }, Date.now() - startTime)
     log.error(err)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code }
+    )
     /**
@@ -273,3 +292,6 @@ * The generation of pack files in the temporary folder (from `os.tmpdir()`)
       log.error(err)
-      incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status })
+      incrementCountMetric(
+        TELEMETRY_GIT_COMMAND_ERRORS,
+        { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code }
+      )
     }
@@ -276,0 +298,0 @@ }

packages/dd-trace/src/plugins/util/test.js

@@ -56,3 +56,4 @@ const path = require('path')
 const TEST_IS_RETRY = 'test.is_retry'
-const TEST_EARLY_FLAKE_IS_ENABLED = 'test.early_flake.is_enabled'
+const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled'
+const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason'
 
@@ -106,3 +107,4 @@ const CI_APP_ORIGIN = 'ciapp-test'
   TEST_IS_RETRY,
-  TEST_EARLY_FLAKE_IS_ENABLED,
+  TEST_EARLY_FLAKE_ENABLED,
+  TEST_EARLY_FLAKE_ABORT_REASON,
   getTestEnvironmentMetadata,
@@ -146,3 +148,4 @@ getTestParametersString,
   removeEfdStringFromTestName,
-  addEfdStringToTestName
+  addEfdStringToTestName,
+  getIsFaultyEarlyFlakeDetection
 }
@@ -577,1 +580,19 @@
 }
+
+function getIsFaultyEarlyFlakeDetection (projectSuites, testsBySuiteName, faultyThresholdPercentage) {
+  let newSuites = 0
+  for (const suite of projectSuites) {
+    if (!testsBySuiteName[suite]) {
+      newSuites++
+    }
+  }
+  const newSuitesPercentage = (newSuites / projectSuites.length) * 100
+
+  // The faulty threshold represents a percentage, but we also want to consider
+  // smaller projects, where big variations in the % are more likely.
+  // This is why we also check the absolute number of new suites.
+  return (
+    newSuites > faultyThresholdPercentage &&
+    newSuitesPercentage > faultyThresholdPercentage
+  )
+}

packages/dd-trace/src/profiling/config.js

@@ -24,2 +24,3 @@ 'use strict'
       DD_PROFILING_CODEHOTSPOTS_ENABLED,
+      DD_PROFILING_CPU_ENABLED,
       DD_PROFILING_DEBUG_SOURCE_MAPS,
@@ -169,3 +170,3 @@ DD_PROFILING_ENABLED,
       DD_PROFILING_TIMELINE_ENABLED,
-      DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, false))
+      DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, samplingContextsAvailable))
     logExperimentalVarDeprecation('TIMELINE_ENABLED')
@@ -181,3 +182,5 @@ checkOptionWithSamplingContextAllowed(this.timelineEnabled, 'Timeline view')
     this.cpuProfilingEnabled = isTrue(coalesce(options.cpuProfilingEnabled,
-      DD_PROFILING_EXPERIMENTAL_CPU_ENABLED, false))
+      DD_PROFILING_CPU_ENABLED,
+      DD_PROFILING_EXPERIMENTAL_CPU_ENABLED, samplingContextsAvailable))
+    logExperimentalVarDeprecation('CPU_ENABLED')
     checkOptionWithSamplingContextAllowed(this.cpuProfilingEnabled, 'CPU profiling')
@@ -294,4 +297,5 @@
 
-  // Events profiler is a profiler for timeline events
-  if (options.timelineEnabled) {
+  // Events profiler is a profiler that produces timeline events. It is only
+  // added if timeline is enabled and there's a wall profiler.
+  if (options.timelineEnabled && profilers.some(p => p instanceof WallProfiler)) {
     profilers.push(new EventsProfiler(options))
@@ -298,0 +302,0 @@ }

packages/dd-trace/src/profiling/exporters/agent.js

@@ -13,2 +13,3 @@ 'use strict'
 const os = require('os')
+const { urlToHttpOptions } = require('url')
 const perf = require('perf_hooks').performance
@@ -181,5 +182,6 @@
         } else {
-          options.protocol = this._url.protocol
-          options.hostname = this._url.hostname
-          options.port = this._url.port
+          const httpOptions = urlToHttpOptions(this._url)
+          options.protocol = httpOptions.protocol
+          options.hostname = httpOptions.hostname
+          options.port = httpOptions.port
         }
@@ -186,0 +188,0 @@

packages/dd-trace/src/profiling/profiler.js

@@ -7,3 +7,6 @@ 'use strict'
 const { threadNamePrefix } = require('./profilers/shared')
+const dc = require('dc-polyfill')
 
+const profileSubmittedChannel = dc.channel('datadog:profiling:profile-submitted')
+
 function maybeSourceMap (sourceMap, SourceMapper, debug) {
@@ -165,2 +168,3 @@ if (!sourceMap) return
       await this._submit(encodedProfiles, startDate, endDate, snapshotKind)
+      profileSubmittedChannel.publish()
       this._logger.debug('Submitted profiles')
@@ -167,0 +171,0 @@ } catch (err) {

packages/dd-trace/src/proxy.js

@@ -14,3 +14,5 @@ 'use strict'
 const dogstatsd = require('./dogstatsd')
+const NoopDogStatsDClient = require('./noop/dogstatsd')
 const spanleak = require('./spanleak')
+const { SSITelemetry } = require('./profiling/ssi-telemetry')
 
@@ -24,3 +26,3 @@ class Tracer extends NoopProxy {
     this._pluginManager = new PluginManager(this)
-    this.dogstatsd = new dogstatsd.NoopDogStatsDClient()
+    this.dogstatsd = new NoopDogStatsDClient()
     this._tracingInitialized = false
@@ -77,2 +79,4 @@ }
 
+      const ssiTelemetry = new SSITelemetry()
+      ssiTelemetry.start()
       if (config.profiling.enabled) {
@@ -86,2 +90,4 @@ // do not stop tracer initialization if the profiler fails to be imported
         }
+      } else if (ssiTelemetry.enabled()) {
+        require('./profiling/ssi-telemetry-mock-profiler').start(config)
       }
@@ -88,0 +94,0 @@ if (!this._profilerStarted) {

packages/dd-trace/src/telemetry/index.js

@@ -13,2 +13,3 @@ 'use strict'
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
+const telemetryAppClosingChannel = dc.channel('datadog:telemetry:app-closing')
 
@@ -133,8 +134,8 @@ let config
   }
+  // Give chance to listeners to update metrics before shutting down.
+  telemetryAppClosingChannel.publish()
   const { reqType, payload } = createPayload('app-closing')
   sendData(config, application, host, reqType, payload)
-  // we flush before shutting down. Only in CI Visibility
-  if (config.isCiVisibility) {
-    metricsManager.send(config, application, host)
-  }
+  // We flush before shutting down.
+  metricsManager.send(config, application, host)
 }
@@ -141,0 +142,0 @@

packages/dd-trace/src/telemetry/metrics.js

@@ -78,4 +78,4 @@ 'use strict'
 
-  dec (value = -1) {
-    return this.track(value)
+  dec (value = 1) {
+    return this.track(-value)
   }
@@ -82,0 +82,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

1695562

increased by1.05%

Number of package files

522

increased by0.58%

Lines of code

49257

increased by0.87%

Worsened metrics

Number of low supply chain risk alerts

154

increased by1.99%

Dependency changes

• + Added@datadog/native-appsec@7.1.1(transitive)

• + Added@datadog/pprof@5.2.0(transitive)

• - Removed@datadog/native-appsec@7.1.0(transitive)

• - Removed@datadog/pprof@5.1.0(transitive)

• Updated@datadog/native-appsec@7.1.1

• Updated@datadog/pprof@5.2.0

• Updatedpprof-format@^2.1.0