Implements Golden Ticket retry mechanism

src/@types/index.ts

@@ -986,6 +986,7 @@ export interface NetworkParameters {
   minTollUsdStr: string
   defaultTollUsdStr: string
   goldenTicketServerUrl: string
+  goldenTicketRetryInterval: number
   dao: {
     proposalFeeUsdStr: string
     voteThresholdUsdStr: string

src/config/index.ts

@@ -104,6 +104,7 @@ export const INITIAL_PARAMETERS: NetworkParameters = {
   minTollUsdStr: '0.2',
   defaultTollUsdStr: '0.2',
   goldenTicketServerUrl: 'http://localhost:3456/golden/ticket',
+  goldenTicketRetryInterval: 10 * ONE_MINUTE,
   messageRetentionDays: 7,
   messageMaxLength: 500,
   dao: {

src/index.ts

@@ -79,6 +79,9 @@ const daoPreCrackTxTypes = new Set([
 
 let isReadyToJoinLatestValue = false
 let mustUseAdminCert = false
+let nextGoldenTicketRetryAt = 0
+let isGoldenTicketRetry = false
+let isGoldenTicketFetchInProgress = false
 
 const shardusSetup = (): void => {
   // SDK SETUP FUNCTIONS
@@ -1575,27 +1578,52 @@ const shardusSetup = (): void => {
 
       isReadyToJoinLatestValue = false
       mustUseAdminCert = false
+      const hasExpiredGoldenTicket = adminCert && adminCert.certExp <= dapp.shardusGetTime() && adminCert?.goldenTicket === true
 
       // query admin cert from the golden ticket server
-      if (!isRequestedAdminCert && networkAccount && utils.isEqualOrNewerVersion('2.4.3', networkAccount.current.activeVersion)) {
+      if (
+        (!isRequestedAdminCert || hasExpiredGoldenTicket) &&
+        networkAccount &&
+        utils.isEqualOrNewerVersion('2.4.3', networkAccount.current.activeVersion) &&
+        dapp.shardusGetTime() >= nextGoldenTicketRetryAt &&
+        !isGoldenTicketFetchInProgress
+      ) {
+        isGoldenTicketFetchInProgress = true
         try {
-          markRequestedAdminCert() // one-time request only
-
-          const goldenTicket = await tryAndFetchGoldenTicket(publicKey, networkAccount, dapp)
-          if (goldenTicket) {
-            setAdminCertificate(goldenTicket)
+          const goldenTicketResult = await tryAndFetchGoldenTicket(publicKey, networkAccount, dapp, isGoldenTicketRetry)
+          if (goldenTicketResult.ticket) {
+            setAdminCertificate(goldenTicketResult.ticket)
+            markRequestedAdminCert()
+            isGoldenTicketRetry = false
             /* prettier-ignore */
-            if (LiberdusFlags.VerboseLogs) console.log(`fetched golden ticket: ${Utils.safeStringify(goldenTicket)}`)
+            if (LiberdusFlags.VerboseLogs) console.log(`fetched golden ticket: ${Utils.safeStringify(goldenTicketResult.ticket)}`)
             nestedCountersInstance.countEvent('liberdus-staking', 'fetched golden ticket from server')
             console.log(`Admin certificate is set to `, adminCert)
+          } else if (goldenTicketResult.terminal) {
+            // An expired ticket otherwise keeps hasExpiredGoldenTicket true and
+            // bypasses isRequestedAdminCert on every isReadyToJoin invocation.
+            if (hasExpiredGoldenTicket) setAdminCertificate(null)
+            markRequestedAdminCert()
+            isGoldenTicketRetry = false
+            /* prettier-ignore */
+            if (LiberdusFlags.VerboseLogs) console.log(`terminal golden ticket fetch error: ${goldenTicketResult.error}`)
+            nestedCountersInstance.countEvent('liberdus-staking', `terminal golden ticket fetch error: ${goldenTicketResult.error}`)
           } else {
+            const goldenTicketRetryInterval = networkAccount.current.goldenTicketRetryInterval || 10 * configs.ONE_MINUTE
+            nextGoldenTicketRetryAt = dapp.shardusGetTime() + goldenTicketRetryInterval
+            isGoldenTicketRetry = true
             /* prettier-ignore */
-            if (LiberdusFlags.VerboseLogs) console.log(`no golden ticket available from server`)
-            nestedCountersInstance.countEvent('liberdus-staking', 'no golden ticket available from server')
+            if (LiberdusFlags.VerboseLogs) console.log(`no golden ticket available from server, retrying in ${goldenTicketRetryInterval}ms`)
+            nestedCountersInstance.countEvent('liberdus-staking', `no golden ticket available from server, retrying in ${goldenTicketRetryInterval}ms`)
           }
         } catch (e) {
+          const goldenTicketRetryInterval = networkAccount.current.goldenTicketRetryInterval || 10 * configs.ONE_MINUTE
+          nextGoldenTicketRetryAt = dapp.shardusGetTime() + goldenTicketRetryInterval
+          isGoldenTicketRetry = true
           /* prettier-ignore */
-          if (logFlags.error) console.log(`Error fetching golden ticket: ${e.message}`) // non fatal
+          if (logFlags.error) console.log(`Error fetching golden ticket: ${e.message}; retrying in ${goldenTicketRetryInterval}ms`) // non fatal
+        } finally {
+          isGoldenTicketFetchInProgress = false
         }
       }
       console.log('is AdminCert set to ', adminCert)

src/transactions/admin_certificate.ts

@@ -22,6 +22,13 @@ export interface AdminCertResponse {
   cached?: boolean
 }
 
+export interface GoldenTicketFetchResult {
+  ticket?: AdminCert
+  error?: string
+  retryable: boolean
+  terminal: boolean
+}
+
 export type PutAdminCertRequest = AdminCert
 
 export interface PutAdminCertResult {
@@ -32,6 +39,33 @@ export interface PutAdminCertResult {
 export let adminCert: AdminCert = null
 export let isRequestedAdminCert: boolean = false
 
+export function isTerminalGoldenTicketError(error?: string): boolean {
+  if (!error) return false
+  const normalizedError = error.toLowerCase()
+  return (
+    normalizedError.includes('schema validation failed') ||
+    normalizedError.includes('invalid request format') ||
+    normalizedError.includes('public key not registered') ||
+    normalizedError.includes('inactive') ||
+    normalizedError.includes('validator not found') ||
+    normalizedError.includes('nonce cannot be empty') ||
+    normalizedError.includes('nonce must be') ||
+    normalizedError.includes('port must be between') ||
+    normalizedError.includes('signature owner does not match registered public key') ||
+    normalizedError.includes('invalid signature') ||
+    normalizedError.includes('signature validation failed')
+  )
+}
+
+function createGoldenTicketFetchResult(error?: string): GoldenTicketFetchResult {
+  const terminal = isTerminalGoldenTicketError(error)
+  return {
+    error,
+    retryable: !terminal,
+    terminal,
+  }
+}
+
 function validatePutAdminCertRequest(req: PutAdminCertRequest, shardus: Shardus): PutAdminCertResult {
   const publicKey = shardus.crypto.getPublicKey()
 
@@ -79,29 +113,42 @@ export async function putAdminCertificateHandler(req: Request, shardus: Shardus)
   return { success: true }
 }
 
-export async function tryAndFetchGoldenTicket(publicKey: string, network: NetworkAccount, dapp: Shardus): Promise<AdminCert> {
+export async function tryAndFetchGoldenTicket(
+  publicKey: string,
+  network: NetworkAccount,
+  dapp: Shardus,
+  isRetry = false,
+): Promise<GoldenTicketFetchResult> {
   try {
     if (LiberdusFlags.VerboseLogs) console.log('Fetching golden ticket from', network.current.goldenTicketServerUrl, 'for publicKey', publicKey, 'node')
     const goldenTicketRequest: any = {
       publicKey,
       ip: config.server.ip.externalIp,
       port: config.server.ip.externalPort,
-      timestamp: dapp.shardusGetTime(),
+      timestamp: dapp.shardusGetTime() + (isRetry ? 1000 : 0),
       nonce: Math.floor(Math.random() * 1e6),
     }
     const signedGoldenTicketRequest: GoldenTicketRequest = dapp.signAsNode(goldenTicketRequest)
     if (LiberdusFlags.VerboseLogs) console.log('Golden Ticket request', signedGoldenTicketRequest)
     const response = await shardusPost<AdminCertResponse>(network.current.goldenTicketServerUrl, signedGoldenTicketRequest, { timeout: 5000 })
     if (LiberdusFlags.VerboseLogs) console.log('Golden Ticket response', response.data)
-    if (response.data && response.data.success) {
-      return response.data.ticket
+    if (response.data && response.data.success && response.data.ticket) {
+      return {
+        ticket: response.data.ticket,
+        retryable: false,
+        terminal: false,
+      }
     } else {
-      console.error('No golden ticket received')
-      return null
+      const error = response.data?.error || 'No golden ticket received'
+      console.error(error)
+      return createGoldenTicketFetchResult(error)
     }
   } catch (error) {
-    console.error(`Error fetching golden ticket from - ${(error as Error).message}`)
-    return null
+    const axiosError = error as { message?: string; response?: { data?: AdminCertResponse; status?: number } }
+    const responseError = axiosError.response?.data?.error
+    const errorMessage = responseError || axiosError.message || 'Unknown Golden Ticket fetch error'
+    console.error(`Error fetching golden ticket from - ${errorMessage}`)
+    return createGoldenTicketFetchResult(errorMessage)
   }
 }
 export function setAdminCertificate(cert: AdminCert): void {

src/transactions/apply_change_network_param.ts

@@ -79,6 +79,9 @@ export function backfillNetworkAccount(network: NetworkAccount): void {
     if (network.current.goldenTicketServerUrl === undefined || network.current.goldenTicketServerUrl === null) {
       network.current.goldenTicketServerUrl = 'http://localhost:3456/golden/ticket'
     }
+    if (network.current.goldenTicketRetryInterval === undefined || network.current.goldenTicketRetryInterval === null) {
+      network.current.goldenTicketRetryInterval = 10 * config.ONE_MINUTE
+    }
     if (network.current.messageRetentionDays === undefined || network.current.messageRetentionDays === null) {
       network.current.messageRetentionDays = 7
     }

test/admin_certificate.test.ts

@@ -0,0 +1,116 @@
+jest.mock('../src/utils/request', () => ({
+  shardusPost: jest.fn(),
+}))
+
+import { Shardus } from '@shardus/core'
+import { shardusPost } from '../src/utils/request'
+import { AdminCert, isTerminalGoldenTicketError, tryAndFetchGoldenTicket } from '../src/transactions/admin_certificate'
+import { NetworkAccount } from '../src/@types'
+
+const mockShardusPost = shardusPost as jest.Mock
+
+const network = {
+  current: {
+    goldenTicketServerUrl: 'http://localhost:3456/golden/ticket',
+  },
+} as NetworkAccount
+
+const dapp = {
+  shardusGetTime: jest.fn(() => 123456),
+  signAsNode: jest.fn((request) => ({ ...request, sign: { owner: request.publicKey, sig: 'signature' } })),
+} as unknown as Shardus
+
+const ticket: AdminCert = {
+  nominee: 'public-key',
+  certCreation: 123456,
+  certExp: 223456,
+  goldenTicket: true,
+  sign: {
+    owner: 'server',
+    sig: 'signature',
+  },
+}
+
+describe('Golden Ticket fetch', () => {
+  beforeEach(() => {
+    mockShardusPost.mockReset()
+    jest.spyOn(console, 'error').mockImplementation(() => undefined)
+  })
+
+  afterEach(() => {
+    jest.restoreAllMocks()
+  })
+
+  it('returns a ticket on a successful Golden Ticket response', async () => {
+    mockShardusPost.mockResolvedValue({ data: { success: true, ticket } })
+
+    const result = await tryAndFetchGoldenTicket('public-key', network, dapp)
+
+    expect(result).toEqual({ ticket, retryable: false, terminal: false })
+  })
+
+  it('classifies timestamp errors from HTTP responses as retryable', async () => {
+    mockShardusPost.mockRejectedValue({
+      message: 'Request failed with status code 409',
+      response: {
+        status: 409,
+        data: {
+          success: false,
+          error: 'Timestamp out of acceptable range. Difference: 10000ms, Tolerance: 5000ms',
+        },
+      },
+    })
+
+    const result = await tryAndFetchGoldenTicket('public-key', network, dapp)
+
+    expect(result.retryable).toBe(true)
+    expect(result.terminal).toBe(false)
+    expect(result.error).toContain('Timestamp out of acceptable range')
+  })
+
+  it('classifies unregistered public keys as terminal', async () => {
+    mockShardusPost.mockRejectedValue({
+      message: 'Request failed with status code 401',
+      response: {
+        status: 401,
+        data: {
+          success: false,
+          error: 'Public key not registered or inactive',
+        },
+      },
+    })
+
+    const result = await tryAndFetchGoldenTicket('public-key', network, dapp)
+
+    expect(result.retryable).toBe(false)
+    expect(result.terminal).toBe(true)
+    expect(result.error).toBe('Public key not registered or inactive')
+  })
+
+  it('classifies network failures as retryable', async () => {
+    mockShardusPost.mockRejectedValue(new Error('connect ECONNREFUSED 127.0.0.1:3456'))
+
+    const result = await tryAndFetchGoldenTicket('public-key', network, dapp)
+
+    expect(result.retryable).toBe(true)
+    expect(result.terminal).toBe(false)
+    expect(result.error).toContain('ECONNREFUSED')
+  })
+
+  it('adds one second to the timestamp when retrying', async () => {
+    mockShardusPost.mockResolvedValue({ data: { success: true, ticket } })
+
+    await tryAndFetchGoldenTicket('public-key', network, dapp, true)
+
+    expect(dapp.signAsNode).toHaveBeenCalledWith(expect.objectContaining({ timestamp: 124456 }))
+  })
+
+  it('identifies terminal Golden Ticket validation errors', () => {
+    expect(isTerminalGoldenTicketError('Public key not registered or inactive')).toBe(true)
+    expect(isTerminalGoldenTicketError('Schema validation failed: publicKey must be a string')).toBe(true)
+    expect(isTerminalGoldenTicketError('Nonce must be a non-negative integer')).toBe(true)
+    expect(isTerminalGoldenTicketError('Port must be between 1 and 65535')).toBe(true)
+    expect(isTerminalGoldenTicketError('Timestamp out of acceptable range')).toBe(false)
+    expect(isTerminalGoldenTicketError('Rate limit exceeded for this validator')).toBe(false)
+  })
+})