Parser Hardening — Implementation Plan

export interface UngroundedItem {
  field: 'commodity_specialisations' | 'certifications' | 'tools_and_systems'
  item: string
}

export interface ParseMetadata {
  source_file: string
  file_type: FileType
  parse_status: ParseStatus
  unreadable_reason: string | null
  overall_confidence: number
  parsed_at: string
  ungrounded_items: UngroundedItem[]
  stale_cv: boolean
  last_role_end: string | null
}

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npx tsc --noEmit

cd "G:/My Drive/BL Dev work/Calibre/calibre-app"
git add src/lib/cv-parser/types.ts
git commit -m "feat(types): extend ParseMetadata with grounding + staleness fields"

import { describe, it, expect } from 'vitest'
import { isGrounded, checkGrounding } from '@/lib/cv-parser/grounding'
import type { ParsedCV } from '@/lib/cv-parser/types'
 
describe('isGrounded', () => {
  it('returns true when all significant tokens appear in source', () => {
    expect(isGrounded('Crude Oil', 'I worked on crude oil trading.')).toBe(true)
  })
 
  it('returns false when no significant token appears in source', () => {
    expect(isGrounded('LNG', 'I worked on fuel oil and bunkers.')).toBe(false)
  })
 
  it('matches short acronyms as whole words only', () => {
    expect(isGrounded('LNG', 'Spot LNG trading desk.')).toBe(true)
    // "LNG" embedded inside "LONGEVITY" should NOT count
    expect(isGrounded('LNG', 'Longevity studies and longitudinal data.')).toBe(false)
  })
 
  it('is case-insensitive for multi-char tokens', () => {
    expect(isGrounded('Bloomberg', 'Built dashboards in BLOOMBERG.')).toBe(true)
  })
 
  it('returns false if any required token is missing', () => {
    // "Crack Spread Products" — significant tokens: crack, spread, products
    // Source only mentions "crack" → ungrounded
    expect(isGrounded('Crack Spread Products', 'Traded crack and other refined products.')).toBe(false)
  })
 
  it('ignores stop words', () => {
    expect(isGrounded('Risk and Controls', 'Built the risk controls framework.')).toBe(true)
  })
 
  it('returns false for empty item', () => {
    expect(isGrounded('', 'anything')).toBe(false)
  })
})
 
const fixture = (): ParsedCV => ({
  identity: {
    name:         { value: 'Test', confidence: 1 },
    email:        { value: null, confidence: null },
    phone:        { value: null, confidence: null },
    location:     { value: null, confidence: null },
    linkedin_url: { value: null, confidence: null },
  },
  current_role: {
    raw_title:  { value: 'Trader', confidence: 1 },
    employer:   { value: 'Acme', confidence: 1 },
    start_date: { value: '2020-01', confidence: 1 },
    end_date:   { value: null, confidence: null },
  },
  employment_history: [],
  signals: {
    total_years_experience:    { value: 5, confidence: 1 },
    seniority_signal:          { value: 'IC', confidence: 1 },
    commodity_specialisations: { value: ['Crude Oil', 'LNG'], confidence: 0.95 },
    vertical_signals:          { value: [], confidence: 1 },
    tools_and_systems:         { value: ['Bloomberg'], confidence: 1 },
    certifications:            { value: [], confidence: 1 },
    education:                 [],
    career_gaps:               { value: [], confidence: 1 },
    short_tenures:             { value: [], confidence: 1 },
  },
  parse_metadata: {
    source_file: 'test.pdf',
    file_type: 'pdf',
    parse_status: 'success',
    unreadable_reason: null,
    overall_confidence: 0.95,
    parsed_at: '2026-05-20T00:00:00Z',
    ungrounded_items: [],
    stale_cv: false,
    last_role_end: null,
  },
})
 
describe('checkGrounding', () => {
  it('clamps confidence to 0.5 when one item is ungrounded', () => {
    const cv = fixture()
    const source = 'Crude oil trader in Singapore using Bloomberg.'
    const result = checkGrounding(cv, source)
    expect(result.clamped.signals.commodity_specialisations.confidence).toBeLessThanOrEqual(0.5)
    expect(result.ungroundedItems).toContainEqual({ field: 'commodity_specialisations', item: 'LNG' })
  })
 
  it('leaves confidence untouched when all items ground', () => {
    const cv = fixture()
    const source = 'Crude oil and LNG trader. Bloomberg user.'
    const result = checkGrounding(cv, source)
    expect(result.clamped.signals.commodity_specialisations.confidence).toBe(0.95)
    expect(result.ungroundedItems).toEqual([])
  })
 
  it('keeps ungrounded items in the value array (does not delete)', () => {
    const cv = fixture()
    const source = 'Crude oil trader using Bloomberg.'
    const result = checkGrounding(cv, source)
    expect(result.clamped.signals.commodity_specialisations.value).toEqual(['Crude Oil', 'LNG'])
  })
})

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npm test tests/lib/cv-parser/grounding.test.ts

import type { ParsedCV, UngroundedItem } from './types'
 
const STOP_WORDS = new Set([
  'the', 'and', 'for', 'with', 'from', 'into',
  'product', 'products', 'service', 'services',
  'system', 'systems', 'group', 'team',
])
 
interface TokenSet {
  longTokens: string[]
  acronyms: string[]
}
 
function tokenise(item: string): TokenSet {
  const cleaned = item.replace(/[^a-zA-Z0-9\s]/g, ' ')
  const words = cleaned.split(/\s+/).filter(Boolean)
  const longTokens: string[] = []
  const acronyms: string[] = []
  for (const w of words) {
    const lower = w.toLowerCase()
    if (STOP_WORDS.has(lower)) continue
    if (w.length >= 4) longTokens.push(lower)
    else if (w.length >= 2) acronyms.push(w)
  }
  return { longTokens, acronyms }
}
 
export function isGrounded(item: string, sourceText: string): boolean {
  const { longTokens, acronyms } = tokenise(item)
  if (longTokens.length === 0 && acronyms.length === 0) return false
 
  const sourceLower = sourceText.toLowerCase()
  for (const t of longTokens) {
    if (!sourceLower.includes(t)) return false
  }
  for (const a of acronyms) {
    const re = new RegExp(`\\b${a.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i')
    if (!re.test(sourceText)) return false
  }
  return true
}
 
export interface GroundingResult {
  clamped: ParsedCV
  ungroundedItems: UngroundedItem[]
}
 
const GROUNDED_FIELDS: Array<UngroundedItem['field']> = [
  'commodity_specialisations',
  'certifications',
  'tools_and_systems',
]
 
export function checkGrounding(cv: ParsedCV, sourceText: string): GroundingResult {
  const ungrounded: UngroundedItem[] = []
 
  for (const field of GROUNDED_FIELDS) {
    const ref = cv.signals[field]
    let anyUngrounded = false
    for (const item of ref.value) {
      if (!isGrounded(item, sourceText)) {
        ungrounded.push({ field, item })
        anyUngrounded = true
      }
    }
    if (anyUngrounded) {
      const current = ref.confidence ?? 0
      ref.confidence = Math.min(current, 0.5)
    }
  }
 
  return { clamped: cv, ungroundedItems: ungrounded }
}

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npm test tests/lib/cv-parser/grounding.test.ts

git add src/lib/cv-parser/grounding.ts tests/lib/cv-parser/grounding.test.ts
git commit -m "feat(parser): add source-grounding check for list fields"

import { describe, it, expect } from 'vitest'
import { checkStaleness } from '@/lib/cv-parser/staleness'
import type { ParsedCV } from '@/lib/cv-parser/types'
 
const baseFixture = (): ParsedCV => ({
  identity: {
    name:         { value: 'Test', confidence: 1 },
    email:        { value: null, confidence: null },
    phone:        { value: null, confidence: null },
    location:     { value: null, confidence: null },
    linkedin_url: { value: null, confidence: null },
  },
  current_role: {
    raw_title:  { value: 'Trader', confidence: 1 },
    employer:   { value: 'Acme', confidence: 1 },
    start_date: { value: '2020-01', confidence: 1 },
    end_date:   { value: null, confidence: null },
  },
  employment_history: [],
  signals: {
    total_years_experience:    { value: 5, confidence: 1 },
    seniority_signal:          { value: 'IC', confidence: 1 },
    commodity_specialisations: { value: [], confidence: 1 },
    vertical_signals:          { value: [], confidence: 1 },
    tools_and_systems:         { value: [], confidence: 1 },
    certifications:            { value: [], confidence: 1 },
    education:                 [],
    career_gaps:               { value: [], confidence: 1 },
    short_tenures:             { value: [], confidence: 1 },
  },
  parse_metadata: {
    source_file: 'test.pdf',
    file_type: 'pdf',
    parse_status: 'success',
    unreadable_reason: null,
    overall_confidence: 0.95,
    parsed_at: '2026-05-20T00:00:00Z',
    ungrounded_items: [],
    stale_cv: false,
    last_role_end: null,
  },
})
 
const NOW = new Date(2026, 4, 20) // 2026-05-20
 
describe('checkStaleness', () => {
  it('returns not-stale when current_role end_date is null', () => {
    const cv = baseFixture()
    const result = checkStaleness(cv, NOW)
    expect(result.staleCV).toBe(false)
    expect(result.lastRoleEnd).toBeNull()
  })
 
  it('returns not-stale when any history role has null end_date', () => {
    const cv = baseFixture()
    cv.current_role.end_date = { value: '2020-12', confidence: 1 }
    cv.employment_history = [
      { raw_title: { value: 'X', confidence: 1 }, employer: { value: 'Y', confidence: 1 },
        start_date: { value: '2021-01', confidence: 1 }, end_date: { value: null, confidence: null },
        description_summary: { value: '', confidence: 1 } },
    ]
    const result = checkStaleness(cv, NOW)
    expect(result.staleCV).toBe(false)
  })
 
  it('flags stale when latest end_date is over 12 months ago (Jonathan Lam case)', () => {
    const cv = baseFixture()
    cv.current_role.end_date = { value: '2019-01', confidence: 1 }
    cv.employment_history = [
      { raw_title: { value: 'X', confidence: 1 }, employer: { value: 'Y', confidence: 1 },
        start_date: { value: '2016-03', confidence: 1 }, end_date: { value: '2016-11', confidence: 1 },
        description_summary: { value: '', confidence: 1 } },
    ]
    const result = checkStaleness(cv, NOW)
    expect(result.staleCV).toBe(true)
    expect(result.lastRoleEnd).toBe('2019-01')
  })
 
  it('does NOT flag stale when latest end_date is within 12 months', () => {
    const cv = baseFixture()
    cv.current_role.end_date = { value: '2025-08', confidence: 1 } // 9 months ago from NOW
    const result = checkStaleness(cv, NOW)
    expect(result.staleCV).toBe(false)
    expect(result.lastRoleEnd).toBe('2025-08')
  })
 
  it('returns null lastRoleEnd when no parseable dates', () => {
    const cv = baseFixture()
    cv.current_role.end_date = { value: 'unknown', confidence: 0.1 }
    const result = checkStaleness(cv, NOW)
    expect(result.staleCV).toBe(false)
    expect(result.lastRoleEnd).toBeNull()
  })
})

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npm test tests/lib/cv-parser/staleness.test.ts

import type { ParsedCV } from './types'
 
const STALE_THRESHOLD_MONTHS = 12
 
function parseYearMonth(ym: string | null): Date | null {
  if (!ym) return null
  const m = ym.match(/^(\d{4})-(\d{2})$/)
  if (!m) return null
  const year = parseInt(m[1], 10)
  const month = parseInt(m[2], 10)
  if (month < 1 || month > 12) return null
  return new Date(year, month - 1, 1)
}
 
function formatYearMonth(d: Date): string {
  return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`
}
 
export interface StalenessResult {
  staleCV: boolean
  lastRoleEnd: string | null
}
 
export function checkStaleness(cv: ParsedCV, now: Date = new Date()): StalenessResult {
  if (cv.current_role.end_date.value === null) {
    return { staleCV: false, lastRoleEnd: null }
  }
  for (const role of cv.employment_history) {
    if (role.end_date.value === null) {
      return { staleCV: false, lastRoleEnd: null }
    }
  }
 
  const dates: Date[] = []
  const currentEnd = parseYearMonth(cv.current_role.end_date.value)
  if (currentEnd) dates.push(currentEnd)
  for (const role of cv.employment_history) {
    const d = parseYearMonth(role.end_date.value)
    if (d) dates.push(d)
  }
 
  if (dates.length === 0) {
    return { staleCV: false, lastRoleEnd: null }
  }
 
  const latest = dates.reduce((a, b) => (a > b ? a : b))
  const cutoff = new Date(now.getFullYear(), now.getMonth() - STALE_THRESHOLD_MONTHS, 1)
  const stale = latest < cutoff
 
  return { staleCV: stale, lastRoleEnd: formatYearMonth(latest) }
}

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npm test tests/lib/cv-parser/staleness.test.ts

git add src/lib/cv-parser/staleness.ts tests/lib/cv-parser/staleness.test.ts
git commit -m "feat(parser): add stale-CV detection"

import { checkGrounding } from './grounding'
import { checkStaleness } from './staleness'

- Do NOT split a continuous tenure at one employer into multiple rows. If a candidate held multiple titles at the same employer with no explicit role-change dates in the text, use the latest title and describe role progression in description_summary.

  const cvWithEmptyMeta: ParsedCV = {
    ...(extracted as Omit<ParsedCV, 'parse_metadata'>),
    parse_metadata: {
      source_file: sourceFile,
      file_type: ext,
      parse_status: 'success',
      unreadable_reason: null,
      overall_confidence: 0,
      parsed_at: new Date().toISOString(),
      ungrounded_items: [],
      stale_cv: false,
      last_role_end: null,
    },
  }
 
  const grounding = checkGrounding(cvWithEmptyMeta, rawText)
  const staleness = checkStaleness(grounding.clamped)
  const overallConfidence = computeOverallConfidence(grounding.clamped)
  const parseStatus = deriveParseStatus(
    grounding.clamped,
    overallConfidence,
    grounding.ungroundedItems.length,
  )
 
  return {
    ...grounding.clamped,
    parse_metadata: {
      ...grounding.clamped.parse_metadata,
      parse_status: parseStatus,
      overall_confidence: overallConfidence,
      parsed_at: new Date().toISOString(),
      ungrounded_items: grounding.ungroundedItems,
      stale_cv: staleness.staleCV,
      last_role_end: staleness.lastRoleEnd,
    },
  }

function deriveParseStatus(
  cv: Omit<ParsedCV, 'parse_metadata'>,
  overallConfidence: number,
  ungroundedCount: number,
): ParseStatus {
  const requiredFields: Array<ConfidenceField<unknown>> = [
    cv.identity.name,
    cv.current_role.raw_title,
    cv.current_role.employer,
    cv.signals.total_years_experience,
  ]
  const hasLowConfidence = requiredFields.some(
    f => f.confidence === null || f.confidence < 0.60
  )
  if (ungroundedCount > 0) return 'partial'
  if (hasLowConfidence || overallConfidence < 0.70) return 'partial'
  return 'success'
}

      ungrounded_items: [],
      stale_cv: false,
      last_role_end: null,

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npm test

git add src/lib/cv-parser/parser.ts
git commit -m "feat(parser): wire in grounding + staleness checks; tighten row-split prompt"

return parseCVText(extraction.text, sourceFile)

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npx tsc --noEmit

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npm test

git add src/lib/cv-parser/index.ts
git commit -m "chore(parser): confirm rawText flows into hardened parser"

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && npm run batch-parse

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && node -e "
const fs=require('fs');
const cases=[
  ['Adrian Tan (Risk, Oil).json',           'LNG hallucination'],
  ['Jonathan Lam (Middle Office, Oil).json','stale CV + Crude Oil hallucination'],
  ['Anand Upretic (Risk, Oil).json',        'commodity completeness'],
  ['Sim Qi Xuan (Cargo Ops, Fuel Oil).json','should remain partial/honest'],
];
for(const [f,note] of cases){
  const j=JSON.parse(fs.readFileSync('outputs/parsed-cvs/'+f,'utf8'));
  console.log('---',f,'---');
  console.log('note:                 ',note);
  console.log('parse_status:         ',j.parse_metadata.parse_status);
  console.log('overall_confidence:   ',j.parse_metadata.overall_confidence);
  console.log('stale_cv:             ',j.parse_metadata.stale_cv);
  console.log('last_role_end:        ',j.parse_metadata.last_role_end);
  console.log('ungrounded_items:     ',JSON.stringify(j.parse_metadata.ungrounded_items));
  console.log('commodity confidence: ',j.signals.commodity_specialisations.confidence);
}
"

cd "G:/My Drive/BL Dev work/Calibre/calibre-app" && node -e "
const fs=require('fs');
const files=fs.readdirSync('outputs/parsed-cvs').filter(f=>f.endsWith('.json'));
const counts={success:0,partial:0,failed:0};
const ungroundedHits=[], staleHits=[];
for(const f of files){
  const j=JSON.parse(fs.readFileSync('outputs/parsed-cvs/'+f,'utf8'));
  counts[j.parse_metadata.parse_status]++;
  if(j.parse_metadata.ungrounded_items.length>0) ungroundedHits.push(f);
  if(j.parse_metadata.stale_cv) staleHits.push(f);
}
console.log('Status:',counts);
console.log('CVs with ungrounded items:',ungroundedHits.length);
console.log('Stale CVs:',staleHits.length);
console.log('Stale list:',staleHits);
"

git add outputs/parsed-cvs/
git commit -m "chore(parsed-cvs): re-batch with hardened parser"

## Parser Calibration Audit (2026-05-20)
 
A 7-CV audit compared parsed JSON against source CVs:
 
- **Floor honesty:** scores ≤0.90 are calibrated — model genuinely hedges.
- **Top is anchored:** scores ≥0.95 frequently mask hallucinations (e.g. Adrian Tan flagged with "LNG" specialisation that appears nowhere in his CV).
- **Hallucinations cluster in list fields:** `commodity_specialisations`, `certifications`, `tools_and_systems`.
- **Stale CVs treated as current:** Jonathan Lam's CV ended Jan 2019 but parser called the SOCAR role `current_role` with confidence 1.0.
 
**Hardening applied (2026-05-20):**
- Deterministic source-grounding check on list fields → ungrounded items keep field confidence ≤ 0.5 and land in `parse_metadata.ungrounded_items`.
- Stale-CV detection at 12-month threshold → `parse_metadata.stale_cv` + `last_role_end`.
- Prompt change to discourage row-fabrication at the same employer.
 
After hardening, batch re-run results (fill in after Task 6 runs):
- Status: { success: TBD, partial: TBD, failed: 1 }
- CVs flagged with ungrounded items: TBD
- Stale CVs: TBD

cd "G:/My Drive/BL Dev work/Calibre" && git -C calibre-app status  # confirm no uncommitted code
# the workspace root is not a git repo; the doc edit is just a file save