feat: redrive piece accept

tools/cli.js

@@ -8,6 +8,7 @@ import { migrateFromD1ToDynamo } from './d1-migration/add-to-dynamo.js'
 import { printD1ProvisionsEmails } from './d1-migration/print-d1-emails.js'
 import { verifyD1DynamoMigration } from './d1-migration/verify-d1-dynamo-migration.js'
 import { getOldestPiecesPendingDeals } from './get-oldest-pieces-pending-deals.js'
+import { redrivePieceAccept } from './redrive-piece-accept.js'
 
 const cli = sade('w3infra-cli')
 
@@ -25,6 +26,10 @@ cli
   .command('follow-filecoin-receipt-chain', 'Follow filecoin receipt chain for a piece')
   .action(followFilecoinReceiptChain)
 
+cli
+  .command('redrive-piece-accept', 'Invoke piece/accept to generate missing receipt')
+  .action(redrivePieceAccept)
+
 cli
   .command('d1-dynamo-migration', 'Run the D1 -> Dynamo migration')
   .action(migrateFromD1ToDynamo)

tools/fetch-metrics-for-space.js

@@ -1,24 +1,19 @@
-import {
-  DynamoDBClient,
-  QueryCommand
-} from '@aws-sdk/client-dynamodb'
+import { QueryCommand } from '@aws-sdk/client-dynamodb'
 import { unmarshall } from '@aws-sdk/util-dynamodb'
 import { mustGetEnv } from '../lib/env.js'
+import { getRegion, getStage } from './lib.js'
+import { getDynamoClient } from '../lib/aws/dynamo.js'
 
 export async function fetchMetricsForSpaceCmd () {
   const {
-    ENV,
     SPACE_DID,
     TABLE_NAME,
   } = getEnv()
+  const stage = getStage()
+  const region = getRegion(stage)
+  const dynamo = getDynamoClient({ region })
 
-  const { client, tableName } = getDynamoDb(
-    TABLE_NAME,
-    ENV,
-    getRegion(ENV)
-  )
-
-  const rows = await getAllTableRows(client, tableName, SPACE_DID)
+  const rows = await getAllTableRows(dynamo, TABLE_NAME, SPACE_DID)
   console.log(`Metrics found for provided space DID: ${rows.length}`)
   for (const row of rows) {
     console.log(`${row.name}: ${row.value}`)
@@ -53,37 +48,7 @@ export async function getAllTableRows (dynamo, tableName, space, options = {}) {
  */
 function getEnv() {
   return {
-    ENV: mustGetEnv('ENV'),
     SPACE_DID: mustGetEnv('SPACE_DID'),
     TABLE_NAME: mustGetEnv('TABLE_NAME'),
   }
 }
-
-/**
- * @param {string} env
- */
-function getRegion (env) {
-  if (env === 'staging') {
-    return 'us-east-2'
-  }
-
-  return 'us-west-2'
-}
-
-/**
- * @param {string} tableName
- * @param {string} env
- * @param {string} region
- */
-function getDynamoDb (tableName, env, region) {
-  const endpoint = `https://dynamodb.${region}.amazonaws.com`
-
-  return {
-    client: new DynamoDBClient({
-      region,
-      endpoint
-    }),
-    tableName: `${env}-w3infra-${tableName}`,
-    endpoint
-  }
-}

tools/follow-filecoin-receipt-chain.js

@@ -1,34 +1,27 @@
 import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront'
 
 import * as DID from '@ipld/dag-ucan/did'
-import { Piece } from '@web3-storage/data-segment'
+import { Aggregate, Piece, Proof } from '@web3-storage/data-segment'
+import archy from 'archy'
 
 import { getServiceSigner } from '../filecoin/service.js'
 import { createPieceTable } from '../filecoin/store/piece.js'
 import { createReceiptStore as createFilecoinReceiptStore } from '../filecoin/store/receipt.js'
 import { mustGetEnv } from '../lib/env.js'
+import { getInvocationBucketName, getPieceTableName, getRegion, getServiceDID, getStage, getWorkflowBucketName } from './lib.js'
 
 export async function followFilecoinReceiptChain () {
-  const {
-    ENV,
-    PIECE_CID,
-    PRIVATE_KEY,
-  } = getEnv()
-
-  const AWS_REGION = getRegion(ENV)
-  const pieceTableName = getPieceTableName(ENV)
-  const invocationBucketName = getInvocationBucketName(ENV)
-  const workflowBucketName = getWorkflowBucketName(ENV)
-  const did = getDid(ENV)
-
-  let id = getServiceSigner({
-    privateKey: PRIVATE_KEY
-  })
-  id = id.withDID(DID.parse(did).did())
-
+  const { PIECE_CID, PRIVATE_KEY } = getEnv()
+  const stage = getStage()
+  const region = getRegion(stage)
+  const pieceTableName = getPieceTableName(stage)
+  const invocationBucketName = getInvocationBucketName(stage)
+  const workflowBucketName = getWorkflowBucketName(stage)
+  const did = DID.parse(getServiceDID(stage)).did()
+  const id = getServiceSigner({ privateKey: PRIVATE_KEY }).withDID(did)
   const pieceInfo = Piece.fromString(PIECE_CID)
-  const receiptStore = createFilecoinReceiptStore(AWS_REGION, invocationBucketName, workflowBucketName)
-  const pieceStore = createPieceTable(AWS_REGION, pieceTableName)
+  const receiptStore = createFilecoinReceiptStore(region, invocationBucketName, workflowBucketName)
+  const pieceStore = createPieceTable(region, pieceTableName)
 
   // Get piece in store
   const getPiece = await pieceStore.get({ piece: pieceInfo.link })
@@ -70,7 +63,7 @@
 
   // Get `piece/accept` receipt
   const pieceAcceptReceiptGet = await receiptStore.get(pieceOfferReceiptGet.ok.fx.join.link())
-  if (pieceAcceptReceiptGet.error) throw new Error('could not find receipt')
+  if (pieceAcceptReceiptGet.error) throw new Error(`could not find receipt: ${pieceOfferReceiptGet.ok.fx.join.link()}`)
   console.log('out:', pieceOfferReceiptGet.ok.out)
 
   if (!pieceAcceptReceiptGet.ok.fx.join) throw new Error('receipt without effect')
@@ -108,70 +101,94 @@
   const filecoinAcceptReceiptGet = await receiptStore.get(filecoinAcceptInvocation.link())
   if (filecoinAcceptReceiptGet.error) throw new Error('could not find receipt')
   console.log('out:', filecoinAcceptReceiptGet.ok.out)
+
+  const filecoinAcceptSuccess = 
+    /** @type {import('@web3-storage/upload-api').FilecoinAcceptSuccess|undefined} */
+    (filecoinAcceptReceiptGet.ok.out.ok)
+
+  if (filecoinAcceptSuccess) {
+    console.log(`Piece: ${filecoinAcceptSuccess.piece}`)
+    console.log(`Aggregate: ${filecoinAcceptSuccess.aggregate}`)
+    console.log(`Deal: ${filecoinAcceptSuccess.aux.dataSource.dealID}`)
+    console.log(`Proof:`)
+    console.log(renderInclusionProof({
+      proof: filecoinAcceptSuccess.inclusion.subtree,
+      piece: pieceInfo,
+      style: 'mini'
+    }))
+  }
 }
 
 /**
  * Get Env validating it is set.
  */
 function getEnv() {
   return {
-    ENV: mustGetEnv('ENV'),
     PIECE_CID: mustGetEnv('PIECE_CID'),
     PRIVATE_KEY: mustGetEnv('PRIVATE_KEY'),
   }
 }
 
-/**
- * @param {string} env
- */
-function getRegion (env) {
-  if (env === 'staging') {
-    return 'us-east-2'
-  }
-
-  return 'us-west-2'
-}
+const MAX_DEPTH = 63
 
+// Adapted from https://github.com/web3-storage/data-segment/blob/e9cdcbf76232e5b92ae1d13f6cf973ec9ab657ef/src/proof.js#L62-L86
 /**
- * @param {string} env
+ * @param {{ proof, piece, style: 'mini'|'midi'|'maxi' }} arg
+ * @returns 
  */
-function getPieceTableName (env) {
-  if (env === 'staging') {
-    return 'staging-w3infra-piece-v2'
+function renderInclusionProof ({ proof, piece, style }) {
+  if (Proof.depth(proof) > MAX_DEPTH) {
+    throw new RangeError('merkle proofs with depths greater than 63 are not supported')
   }
 
-  return 'prod-w3infra-piece-v2'
-}
-
-/**
- * @param {string} env
- */
-function getInvocationBucketName (env) {
-  if (env === 'staging') {
-    return 'invocation-store-staging-0'
+  let position = BigInt(Proof.offset(proof))
+  if (position >> BigInt(Proof.depth(proof)) !== 0n) {
+    throw new RangeError('offset greater than width of the tree')
   }
 
-  return 'invocation-store-prod-0'
-}
-
-/**
- * @param {string} env
- */
-function getWorkflowBucketName (env) {
-  if (env === 'staging') {
-    return 'workflow-store-staging-0'
+  const { root } = piece
+  /** @type {archy.Data['nodes']} */
+  let nodes = []
+  let top = root
+  let right = 0n
+  let height = piece.height
+
+  for (const node of Proof.path(proof)) {
+    right =  position & 1n
+    position = position >> 1n
+
+    const label = top === root
+      ? Piece.toLink(piece).toString()
+      : Piece.toLink({ root: top, height: height + 1, padding: 0n }).toString()
+    const otherLabel = Piece.toLink({ root: node, height, padding: 0n }).toString()
+
+    if (style === 'midi' || style === 'maxi') {
+      if (right === 1n) {
+        nodes = [{
+          label: otherLabel,
+          nodes: style === 'maxi' ? ['...', '...'] : []
+        }, {
+          label: `*${label}`,
+          nodes
+        }]
+      } else {
+        nodes = [{
+          label: `*${label}`,
+          nodes
+        }, {
+          label: otherLabel,
+          nodes: style === 'maxi' ? ['...', '...'] : []
+        }]
+      }
+    } else {
+      nodes = [{ label: `*${label}`, nodes }]
+    }
+    top = right === 1n ? Proof.computeNode(node, top) : Proof.computeNode(top, node)
+    height++
   }
 
-  return 'workflow-store-prod-0'
-}
-
-/**
- * @param {string} env
- */
-function getDid (env) {
-  if (env === 'staging') {
-    return 'did:web:staging.web3.storage'
-  }
+  const aggregate = Aggregate.toLink({ root: top, height })
+  const data = { label: aggregate.toString(), nodes }
 
-  return 'did:web:web3.storage'
+  return archy(data)
 }