feat: max-permission-denied guard to halt runs on repeated auth failures

.github/lychee.toml

@@ -18,6 +18,7 @@ exclude = [
   "^https://mcp\\.tavily\\.com",
   "^https://docs\\.sigstore\\.dev/cosign/installation",
   "^https://docs\\.github\\.com/en/github/site-policy/github-bug-bounty",
+  "^https://www\\.sei\\.cmu\\.edu/",
   # LOGGING.md is referenced but doesn't exist yet (planned doc)
   "LOGGING\\.md",
 ]

containers/api-proxy/guards/max-permission-denied-guard.js

@@ -0,0 +1,93 @@
+'use strict';
+
+const { parsePositiveInteger } = require('./guard-utils');
+
+let permDeniedGuardState = {
+  configKey: null,
+  deniedCount: 0,
+};
+
+const permDeniedConfigCache = {
+  rawMax: undefined,
+  parsed: null,
+};
+
+function getPermDeniedConfig() {
+  const rawMax = process.env.AWF_MAX_PERMISSION_DENIED;
+  if (permDeniedConfigCache.rawMax === rawMax) {
+    return permDeniedConfigCache.parsed;
+  }
+  permDeniedConfigCache.rawMax = rawMax;
+  permDeniedConfigCache.parsed = parsePositiveInteger(rawMax);
+  return permDeniedConfigCache.parsed;
+}
+
+function getPermDeniedState(max) {
+  if (!max) return null;
+  const configKey = String(max);
+  if (permDeniedGuardState.configKey !== configKey) {
+    permDeniedGuardState = { configKey, deniedCount: 0 };
+  }
+  return permDeniedGuardState;
+}
+
+function applyPermissionDenied() {
+  const max = getPermDeniedConfig();
+  const state = getPermDeniedState(max);
+  if (!state) return;
+  state.deniedCount += 1;
+}
+
+function getPermissionDeniedBlockState() {
+  const max = getPermDeniedConfig();
+  const state = getPermDeniedState(max);
+  if (!state) return null;
+  return {
+    maxPermissionDenied: max,
+    deniedCount: state.deniedCount,
+    maxExceeded: state.deniedCount >= max,
+  };
+}
+
+function getPermissionDeniedReflectState() {
+  const max = getPermDeniedConfig();
+  const state = getPermDeniedState(max);
+  if (!state) {
+    return {
+      enabled: false,
+      max_permission_denied: null,
+      denied_count: 0,
+    };
+  }
+  return {
+    enabled: true,
+    max_permission_denied: max,
+    denied_count: state.deniedCount,
+  };
+}
+
+function resetPermissionDeniedGuardForTests() {
+  permDeniedGuardState = { configKey: null, deniedCount: 0 };
+  permDeniedConfigCache.rawMax = undefined;
+  permDeniedConfigCache.parsed = null;
+}
+
+function buildPermissionDeniedLimitError(state) {
+  return {
+    error: {
+      type: 'permission_denied_limit_exceeded',
+      message: `Permission denied limit exceeded (${state.deniedCount} / ${state.maxPermissionDenied}). ` +
+        'The run has been stopped due to repeated permission errors — check that all API keys and tokens are correctly configured.',
+      denied_count: state.deniedCount,
+      max_permission_denied: state.maxPermissionDenied,
+    },
+  };
+}
+
+module.exports = {
+  applyPermissionDenied,
+  getPermissionDeniedBlockState,
+  getPermissionDeniedReflectState,
+  resetPermissionDeniedGuardForTests,
+  buildPermissionDeniedLimitError,
+};

containers/api-proxy/guards/max-permission-denied-guard.test.js

@@ -0,0 +1,156 @@
+'use strict';
+
+const {
+  applyPermissionDenied,
+  getPermissionDeniedBlockState,
+  getPermissionDeniedReflectState,
+  resetPermissionDeniedGuardForTests,
+  buildPermissionDeniedLimitError,
+} = require('./max-permission-denied-guard');
+
+describe('max-permission-denied-guard', () => {
+  beforeEach(() => {
+    delete process.env.AWF_MAX_PERMISSION_DENIED;
+    resetPermissionDeniedGuardForTests();
+  });
+
+  afterEach(() => {
+    delete process.env.AWF_MAX_PERMISSION_DENIED;
+    resetPermissionDeniedGuardForTests();
+  });
+
+  describe('when AWF_MAX_PERMISSION_DENIED is not configured', () => {
+    it('applyPermissionDenied does nothing', () => {
+      applyPermissionDenied();
+      expect(getPermissionDeniedBlockState()).toBeNull();
+    });
+
+    it('getPermissionDeniedBlockState returns null', () => {
+      expect(getPermissionDeniedBlockState()).toBeNull();
+    });
+
+    it('getPermissionDeniedReflectState returns disabled state', () => {
+      expect(getPermissionDeniedReflectState()).toEqual({
+        enabled: false,
+        max_permission_denied: null,
+        denied_count: 0,
+      });
+    });
+  });
+
+  describe('when AWF_MAX_PERMISSION_DENIED is configured', () => {
+    beforeEach(() => {
+      process.env.AWF_MAX_PERMISSION_DENIED = '3';
+      resetPermissionDeniedGuardForTests();
+    });
+
+    it('starts with denied_count of 0 and maxExceeded false', () => {
+      const state = getPermissionDeniedBlockState();
+      expect(state).toEqual({
+        maxPermissionDenied: 3,
+        deniedCount: 0,
+        maxExceeded: false,
+      });
+    });
+
+    it('increments denied count on each applyPermissionDenied call', () => {
+      applyPermissionDenied();
+      applyPermissionDenied();
+      const state = getPermissionDeniedBlockState();
+      expect(state.deniedCount).toBe(2);
+      expect(state.maxExceeded).toBe(false);
+    });
+
+    it('sets maxExceeded true when denied count reaches the max', () => {
+      applyPermissionDenied();
+      applyPermissionDenied();
+      applyPermissionDenied();
+      const state = getPermissionDeniedBlockState();
+      expect(state.deniedCount).toBe(3);
+      expect(state.maxExceeded).toBe(true);
+    });
+
+    it('remains exceeded after further denials beyond the limit', () => {
+      for (let i = 0; i < 5; i++) applyPermissionDenied();
+      const state = getPermissionDeniedBlockState();
+      expect(state.deniedCount).toBe(5);
+      expect(state.maxExceeded).toBe(true);
+    });
+
+    it('returns enabled reflect state with running count', () => {
+      applyPermissionDenied();
+      expect(getPermissionDeniedReflectState()).toEqual({
+        enabled: true,
+        max_permission_denied: 3,
+        denied_count: 1,
+      });
+    });
+
+    it('resets state correctly between tests', () => {
+      applyPermissionDenied();
+      resetPermissionDeniedGuardForTests();
+      const state = getPermissionDeniedBlockState();
+      expect(state.deniedCount).toBe(0);
+      expect(state.maxExceeded).toBe(false);
+    });
+  });
+
+  describe('when AWF_MAX_PERMISSION_DENIED is invalid', () => {
+    it('treats zero as unconfigured', () => {
+      process.env.AWF_MAX_PERMISSION_DENIED = '0';
+      resetPermissionDeniedGuardForTests();
+      expect(getPermissionDeniedBlockState()).toBeNull();
+    });
+
+    it('treats non-numeric value as unconfigured', () => {
+      process.env.AWF_MAX_PERMISSION_DENIED = 'abc';
+      resetPermissionDeniedGuardForTests();
+      expect(getPermissionDeniedBlockState()).toBeNull();
+    });
+
+    it('treats negative value as unconfigured', () => {
+      process.env.AWF_MAX_PERMISSION_DENIED = '-1';
+      resetPermissionDeniedGuardForTests();
+      expect(getPermissionDeniedBlockState()).toBeNull();
+    });
+  });
+
+  describe('buildPermissionDeniedLimitError', () => {
+    it('builds a structured error payload', () => {
+      const state = { deniedCount: 3, maxPermissionDenied: 3 };
+      const error = buildPermissionDeniedLimitError(state);
+      expect(error).toEqual({
+        error: {
+          type: 'permission_denied_limit_exceeded',
+          message: expect.stringContaining('3 / 3'),
+          denied_count: 3,
+          max_permission_denied: 3,
+        },
+      });
+    });
+
+    it('includes investigation hint in the message', () => {
+      const state = { deniedCount: 2, maxPermissionDenied: 2 };
+      const error = buildPermissionDeniedLimitError(state);
+      expect(error.error.message).toMatch(/check/i);
+    });
+  });
+
+  describe('config cache invalidation', () => {
+    it('picks up a new AWF_MAX_PERMISSION_DENIED value at runtime', () => {
+      process.env.AWF_MAX_PERMISSION_DENIED = '2';
+      resetPermissionDeniedGuardForTests();
+
+      applyPermissionDenied();
+      applyPermissionDenied();
+      expect(getPermissionDeniedBlockState().maxExceeded).toBe(true);
+
+      // Raise the limit while the process is running
+      process.env.AWF_MAX_PERMISSION_DENIED = '5';
+      const state = getPermissionDeniedBlockState();
+      expect(state.deniedCount).toBe(0);
+      expect(state.maxPermissionDenied).toBe(5);
+      expect(state.maxExceeded).toBe(false);
+    });
+  });
+});

containers/api-proxy/management.js

@@ -29,6 +29,7 @@ const metrics = require('./metrics');
  * @property {() => Record<string, { enabled: boolean, strategy: string, suppressed: boolean, suppression_reason?: string }>} getEffectiveModelFallback - Returns provider-effective fallback summary
  * @property {() => object}         getEffectiveTokenUsage - Returns effective token usage summary
  * @property {() => object}         getMaxRunsUsage        - Returns max-runs usage summary
+ * @property {() => object}         getPermissionDeniedUsage - Returns permission-denied usage summary
  */
 
 /**
@@ -52,6 +53,7 @@ function createManagementHandlers(deps) {
     getEffectiveModelFallback,
     getEffectiveTokenUsage,
     getMaxRunsUsage,
+    getPermissionDeniedUsage,
   } = deps;
 
   /**
@@ -103,6 +105,7 @@ function createManagementHandlers(deps) {
       model_fallback_effective: getEffectiveModelFallback(),
       effective_tokens: getEffectiveTokenUsage(),
       runs: getMaxRunsUsage(),
+      permission_denied: getPermissionDeniedUsage(),
     };
   }
 

containers/api-proxy/proxy-request.js

@@ -39,6 +39,13 @@ const {
   resetMaxRunsGuardForTests,
   buildMaxRunsExceededError,
 } = require('./guards/max-runs-guard');
+const {
+  applyPermissionDenied,
+  getPermissionDeniedBlockState,
+  getPermissionDeniedReflectState,
+  resetPermissionDeniedGuardForTests,
+  buildPermissionDeniedLimitError,
+} = require('./guards/max-permission-denied-guard');
 const {
   getAndClearPendingTimeoutSteeringMessage,
   resetTimeoutSteeringForTests,
@@ -179,6 +186,8 @@ const proxyWebSocket = createProxyWebSocket({
   buildEffectiveTokenLimitError,
   getMaxRunsBlockState,
   buildMaxRunsExceededError,
+  getPermissionDeniedBlockState,
+  buildPermissionDeniedLimitError,
   trackWebSocketTokenUsage,
   applyEffectiveTokenUsage,
 });
@@ -247,6 +256,7 @@ const { handleUpstreamResponse } = createUpstreamResponseHandlers({
   trackTokenUsage,
   applyEffectiveTokenUsage,
   applyMaxRunsInvocation,
+  applyPermissionDenied,
   extractBillingHeaders,
   parseDeprecatedHeaderFromBody,
   learnAndStripDeprecatedHeaderValue,
@@ -504,6 +514,24 @@ function proxyRequest(req, res, targetHost, injectHeaders, provider, basePath =
       return;
     }
 
+    const pdBlock = getPermissionDeniedBlockState();
+    if (pdBlock && pdBlock.maxExceeded) {
+      const duration = Date.now() - startTime;
+      metrics.gaugeDec('active_requests', { provider });
+      metrics.increment('requests_total', { provider, method: req.method, status_class: '4xx' });
+      metrics.observe('request_duration_ms', duration, { provider });
+      logRequest('warn', 'permission_denied_limit_exceeded', {
+        request_id: requestId,
+        provider,
+        denied_count: pdBlock.deniedCount,
+        max_permission_denied: pdBlock.maxPermissionDenied,
+      });
+      otel.endSpan(span, 403);
+      res.writeHead(403, { 'Content-Type': 'application/json', 'X-Request-ID': requestId });
+      res.end(JSON.stringify(buildPermissionDeniedLimitError(pdBlock)));
+      return;
+    }
+
     sendUpstreamRequest(headers, {
       body, targetHost, upstreamPath, req, res, provider, requestId, startTime, span, requestBytes,
     });
@@ -521,8 +549,10 @@ module.exports = {
   HTTPS_PROXY,
   getEffectiveTokenReflectState,
   getMaxRunsReflectState,
+  getPermissionDeniedReflectState,
   resetEffectiveTokenGuardForTests,
   resetMaxRunsGuardForTests,
+  resetPermissionDeniedGuardForTests,
   resetTimeoutSteeringForTests,
   resetAnthropicDeprecatedBetaHeadersForTests: resetDeprecatedHeaderValuesForTests,
   getAndClearPendingSteeringMessage,

containers/api-proxy/server.js

@@ -49,6 +49,7 @@ const {
   extractBillingHeaders,
   getEffectiveTokenReflectState,
   getMaxRunsReflectState,
+  getPermissionDeniedReflectState,
 } = require('./proxy-request');
 
 const {
@@ -125,6 +126,7 @@ const { healthResponse, reflectEndpoints, handleManagementEndpoint } = createMan
   getEffectiveModelFallback: () => getEffectiveModelFallbackForReflect(registeredAdapters),
   getEffectiveTokenUsage: () => getEffectiveTokenReflectState(),
   getMaxRunsUsage: () => getMaxRunsReflectState(),
+  getPermissionDeniedUsage: () => getPermissionDeniedReflectState(),
 });
 
 function buildModelsJson() {