"""Expose existing demonstration assets without changing model results.
Run directly to build the catalogs and install page links. Exports are projections
of existing public payloads or byte-identical copies from replication packages.
"""
from pathlib import Path
import base64,csv,gzip,hashlib,io,json,shutil,zipfile
from html import escape
from bs4 import BeautifulSoup
import openpyxl
ROOT=Path(__file__).resolve().parents[3]
OUT=ROOT/'output/research_presentation'
DEST=OUT/'downloads/application-data'

def dump(name,data):
 (DEST/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n')
def csvwrite(name,rows):
 if not rows:return
 fields=list(rows[0])
 with (DEST/name).open('w',newline='') as f:
  w=csv.DictWriter(f,fieldnames=fields);w.writeheader()
  for row in rows:w.writerow({k:json.dumps(v,ensure_ascii=False) if isinstance(v,(list,dict)) else v for k,v in row.items()})
def catalog(name,title,description,items,note):
 links=''.join(f'<tr><td><a href="{href}" download>{escape(label)}</a></td><td>{escape(desc)}</td></tr>' for label,href,desc in items)
 (DEST/(name+'.html')).write_text('<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>'+title+'</title><style>body{font:17px/1.6 system-ui;color:#172c3d;max-width:1000px;padding:24px;margin:auto}a{color:#24547c}td,th{padding:14px;border-bottom:1px solid #ccd5dd;text-align:left}table{border-collapse:collapse;width:100%}small{color:#526570}</style></head><body><a href="../../index.html#paper-applications">← Applications</a><h1>'+title+'</h1><p>'+description+'</p><table><thead><tr><th>Download</th><th>Contents</th></tr></thead><tbody>'+links+'</tbody></table><p>'+note+'</p><p><a href="README.txt">Field guide and provenance</a> · <a href="export_application_assets.py">Export code</a> · <a href="manifest.json">Checksums</a></p><small>File descriptions generated by Codex. Existing data and review decisions are preserved.</small></body></html>')
def insert(path,html,selector='main',after_heading=True):
 p=ROOT/path;s=BeautifulSoup(p.read_text(),'html.parser');old=s.find(id='application-downloads')
 if old:old.decompose()
 block=BeautifulSoup('<section id="application-downloads" style="padding:18px;margin:18px 0;border:1px solid #ccd5dd;background:#f5f7f9;color:#172c3d">'+html+'</section>','html.parser').section
 target=s.select_one(selector)
 assert target,(path,selector)
 if after_heading and target.find('h1'):target.find('h1').insert_after(block)
 else:target.insert(0,block)
 p.write_text(str(s))
def alternative_html():
 return '<h2>Download the comparison results</h2><p><a href="../research_presentation/downloads/application-data/alternatives.html">Result tables, field guide and replication packages</a></p><p><a href="../research_presentation/downloads/application-data/linked_results.csv" download>Task-linked skills and abilities · CSV</a> · <a href="../research_presentation/downloads/application-data/activity_results.csv" download>Work activities · CSV</a> · <a href="../research_presentation/downloads/application-data/skillmatch_results.csv" download>Direct ESCO SkillMatch comparison · CSV</a></p><p>Capture and share columns are proportions (multiply by 100 for percentages). SkillMatch uses its paired extraction sample.</p>'
def install_alternative(out):
 p=Path(out)/'skills_abilities.html'
 if not p.exists():return
 s=BeautifulSoup(p.read_text(),'html.parser');old=s.find(id='application-downloads')
 if old:old.decompose()
 block=BeautifulSoup('<section id="application-downloads">'+alternative_html()+'</section>','html.parser').section
 s.find('h1').insert_after(block);p.write_text(str(s))
def main():
 DEST.mkdir(parents=True,exist_ok=True)
 sources={};counts={}
 def record(path):sources[str(path.relative_to(ROOT))]=hashlib.sha256(path.read_bytes()).hexdigest()
 for signal in ['stem','green']:
  p=OUT/'us-lmi/downloads/agreement'/f'sonnet_gpt_{signal}_tasks_results.xlsx';record(p)
  shutil.copy2(p,DEST/p.name)
  w=openpyxl.load_workbook(p,read_only=True,data_only=True)
  fields=['task_id','task_text','signal','in_canonical_onet','claude_sonnet_4_6_verdict','gpt_4o_verdict','included','prompt_version']
  for sheet,label in [('agreement_retained','tasks'),('reviewed','review_decisions')]:
   values=iter(w[sheet].values);headers=next(values);rows=[dict(zip(headers,r)) for r in values]
   selected=[{k:r[k] for k in fields} for r in rows];csvwrite(f'{signal}_{label}.csv',selected)
   counts[f'{signal}_{label}']=len(selected)
   if label=='tasks':assert all(r['included']==1 and r['claude_sonnet_4_6_verdict']=='YES' and r['gpt_4o_verdict']=='YES' for r in selected)
  w.close()
 items=[]
 for signal in ['stem','green']:
  items += [(f'{signal.upper()} task list · CSV',f'{signal}_tasks.csv',f"{counts[signal+'_tasks']:,} agreement-retained rows: task IDs, descriptions and both review labels."),(f'{signal.upper()} all decisions · CSV',f'{signal}_review_decisions.csv','Selected review columns for all candidates; not all rows belong to the bundle.'),(f'{signal.upper()} original workbook',f'sonnet_gpt_{signal}_tasks_results.xlsx','Unmodified earlier-dashboard workbook; includes additional review sheets and historical aggregate columns.')]
 catalog('bundles','STEM and green task lists','Download the task membership lists used in the earlier dashboard. These are task-level labels, not classifications of occupations.',items,'Derived CSVs select existing workbook columns; no tasks were reclassified. See the workbook metadata for the source procedure and version.')
 p=ROOT/'analysis/public_source_replication/web/data/military_pathways.json';record(p);d=json.loads(p.read_text());shutil.copy2(p,DEST/'military_pathways.json')
 rows=[]
 for e in d['entries']:
  for category in ['connections','exploratory']:
   for r in e[category]:rows.append(dict(military_id=e['id'],military_code=e['code'],military_title=e['title'],branch=e['branch'],release_category='documented' if category=='connections' else 'exploratory',civilian_soc=r['soc'],civilian_title=r['soc_title'],review_status=r.get('status'),raw_cosine=r.get('raw_cosine'),reviewed_cosine=r.get('reviewed_cosine'),review_note=r.get('review_note'),source_url=e.get('source_url')))
 csvwrite('military_connections.csv',rows);counts['military_connection_rows']=len(rows)
 catalog('military','Military titles: data downloads','The navigator’s exact public payload and a flat table of displayed connections.', [('Connections · CSV','military_connections.csv',f'{len(rows):,} connection rows, including a documented/exploratory category, military and civilian codes, cosine scores and review notes.'),('Complete navigator data · JSON','military_pathways.json',f"{len(d['entries']):,} military title entries, including titles with no displayed connections; shared-task evidence, routes, source URLs and review status.")], 'Documented correspondences are code links, not observed veteran placements. Exploratory connections are not supported pathways; zero additional pathways passed the release support review.')
 items=[]
 for name,label in [('linked','Task-linked skills and abilities'),('activity','Work activities'),('skillmatch','Direct ESCO SkillMatch')]:
  p=ROOT/'output/public_occupation_dashboard'/f'{name}_replication.zip';record(p)
  with zipfile.ZipFile(p) as z:raw=z.read('expected_summary.csv')
  if name=='activity':
   rows=list(csv.DictReader(io.StringIO(raw.decode())))
   csvwrite('activity_results.csv',[r for r in rows if not r['model'].startswith('soft_')])
  else:(DEST/f'{name}_results.csv').write_bytes(raw)
  items += [(label+' · CSV',f'{name}_results.csv','Published result rows from the replication package; the work-activity table includes only tasks, DWA, IWA and GWA.'),(label+' · code and inputs',f'../../../public_occupation_dashboard/{name}_replication.zip','Original runnable package, including aggregate inputs and verification targets.')]
 catalog('alternatives','Alternative representations: result tables','Occupational-transition comparisons. Each file retains the evaluation universe of its own experiment.',items,'mean_capture and share columns are proportions. Differences are proportion differences; multiply by 100 for percentage points. origins is the number of evaluated origin occupations; mean_k is mean comparison-list length. The SkillMatch task baseline uses paired extraction data and can differ from the main task baseline.')
 p=OUT/'calhr/index.html';s=BeautifulSoup(p.read_text(),'html.parser')
 payload=s.find(id='payload').text;d=json.loads(gzip.decompress(base64.b64decode(payload)));sources['calhr_embedded_payload_sha256']=hashlib.sha256(payload.encode()).hexdigest()
 csvwrite('calhr_descriptions.csv',[dict(doc_index=i,doc_id=r['id'],title=r['title'],class_codes=r['codes']) for i,r in enumerate(d['docs'])])
 gaps=[dict(doc_index=g['doc'],doc_id=d['docs'][g['doc']]['id'],title=d['docs'][g['doc']]['title'],normalized_sentence=g.get('normalized_sentence'),source_quote=g.get('source_quote'),classifier_score=g.get('classifier_score'),match_threshold=g.get('match_threshold'),status=g.get('status')) for g in d['gaps']]
 csvwrite('calhr_task_gaps.csv',gaps)
 dump('calhr_run_manifest.json',d['manifest'])
 dump('calhr_coverage_summary.json',d['comparison']['summary'])
 # Preserve lookup tables alongside nested evidence indices so the exported data remain interpretable.
 for name,keys in [('calhr_features',['docs','features','taskLabels','occupationLabels','manifest']),('calhr_taxonomy_comparison',['docs','taskLabels','comparison','crosswalk','singapore','singaporeTasks','manifest'])]:
  raw=json.dumps({k:d[k] for k in keys},ensure_ascii=False,separators=(',',':')).encode()
  (DEST/(name+'.json.gz')).write_bytes(gzip.compress(raw,mtime=0))
 counts['calhr_gap_occurrences']=len(gaps);counts['calhr_descriptions']=len(d['docs'])
 catalog('calhr','CalHR: data downloads','Static exports of the same data embedded in the classification explorer. They can be downloaded without running the application.',[
 ('Description index · CSV','calhr_descriptions.csv',f"{len(d['docs']):,} descriptions, class codes and stable document IDs; doc_index resolves numeric document references in the JSON files."),
 ('Unmatched task statements · CSV','calhr_task_gaps.csv',f'{len(gaps):,} occurrences, including source quotations and classifier scores; repeated statements can occur in different descriptions. Candidates for review, not confirmed new tasks.'),
 ('Extracted feature lists · JSON.gz','calhr_features.json.gz','All feature stages, labels, document lookup and run manifest from the embedded explorer.'),
 ('O*NET / Singapore comparison · JSON.gz','calhr_taxonomy_comparison.json.gz','Task and ability records, source-coverage units, crosswalk pairs, evidence indices and required document/task lookups.'),
 ('Coverage summary · JSON','calhr_coverage_summary.json','Counts of recognized duty units covered by each taxonomy; coverage is source-span overlap, not semantic completeness.'),
 ('Run manifest · JSON','calhr_run_manifest.json','Model versions, stage settings and original run provenance.')], 'For selected subsets, use the explorer’s existing Export buttons. Task gaps are retrieval gaps, not automatically new task concepts. Methods and code remain linked from the application gateway.')
 guide='''APPLICATION DATA: FILE GUIDE AND PROVENANCE

These downloads expose existing public demonstration data. No model was refitted and no review decisions were changed. This guide was generated by Codex.

STEM / GREEN
The *_tasks.csv files select rows from agreement_retained in the original workbooks; *_review_decisions.csv select rows from reviewed. Columns: task_id and task_text (task identity), signal (STEM/green), in_canonical_onet (original source label), both named model verdicts, included (existing bundle decision), prompt_version. Both model verdicts are YES for retained rows. Original workbooks are unchanged. Historical aggregate columns in those workbooks are not part of the current occupational-relatedness sample.

MILITARY
military_connections.csv flattens connections and exploratory from military_pathways.json. release_category preserves that distinction; review_status/review_note retain the existing assessment. Scores are task-profile cosine similarities, not placement probabilities. Military and SOC codes should be imported as strings. The JSON includes entries with no connection rows, source URLs and nested shared-task evidence. It is the exact navigator payload.

ALTERNATIVE REPRESENTATIONS
Linked and SkillMatch result CSVs are byte-identical expected_summary.csv exports. activity_results.csv selects task, DWA, IWA and GWA rows from that file; the historical archive retains other exploratory calculations. dataset identifies the benchmark; model identifies the representation; origins counts evaluated origins; mean_k is average equal-length recommendation count; mean_capture is mean observed-transition recovery (proportion). share_* fields are proportions; mean_difference_* fields are differences in proportions (multiply by 100 for percentage points); origins_higher_* and ties_* are counts. Model keys: task=posting tasks; onet_published=published O*NET; skills/abilities=O*NET task-linked representations; task_plus_*=task-linked blends; dwa/iwa/gwa=work-activity levels; skill=direct ESCO SkillMatch; combined=task/SkillMatch blend. Evaluation samples and benchmark coverage differ by experiment. Do not combine files into a single common-sample result.

CALHR
Exports derive from the explorer's gzip/base64 payload, not a rerun. Numeric doc/doc_index values are zero-based indices into docs. calhr_descriptions.csv supplies stable IDs and titles. The compressed JSON files preserve nested evidence and corresponding lookup structures. calhr_task_gaps.csv includes occurrences rather than unique statements, and preserves original quotations/scores. Gaps are not validated new concepts. Coverage is recognized source-unit overlap. Consult calhr_run_manifest.json for original settings.

CODE AND CHECKS
export_application_assets.py produces these files from the existing public pages/packages/workbooks; it requires the original workspace layout. manifest.json records source hashes, output hashes and row counts. It does not claim to rerun underlying extraction or review. Use each application's existing source and methods links for the full original procedure.
'''
 (DEST/'README.txt').write_text(guide);shutil.copy2(Path(__file__),DEST/'export_application_assets.py')
 dump('manifest.json',{'source_hashes':sources,'counts':counts,'files':{p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(DEST.iterdir()) if p.is_file() and p.name!='manifest.json'}})
 insert('output/research_presentation/bundles.html','<h2>Download STEM and green task lists</h2><p><a href="downloads/application-data/bundles.html">Task lists, both model decisions and original workbooks</a></p><p><a href="downloads/application-data/stem_tasks.csv" download>STEM task list · CSV</a> · <a href="downloads/application-data/green_tasks.csv" download>Green task list · CSV</a></p>',selector='#stem',after_heading=False)
 insert('analysis/public_source_replication/web/military.html','<h2>Download the connections</h2><p><a href="../../../output/research_presentation/downloads/application-data/military.html">Data files and field guide</a> · <a href="../../../output/research_presentation/downloads/application-data/military_connections.csv" download>Connections · CSV</a> · <a href="data/military_pathways.json" download>Complete navigator data · JSON</a></p>')
 insert('output/research_presentation/calhr/index.html','<h2>Download the CalHR data</h2><p><a href="../downloads/application-data/calhr.html">Description index, task gaps, extracted features, O*NET / Singapore comparison and run manifest</a></p><p><a href="../downloads/application-data/calhr_task_gaps.csv" download>Unmatched task statements · CSV</a> · <a href="../downloads/application-data/calhr_descriptions.csv" download>Description index · CSV</a></p>',selector='body',after_heading=False)
 install_alternative(ROOT/'output/public_occupation_dashboard')
 print(json.dumps(counts));print('Assets MB',round(sum(p.stat().st_size for p in DEST.iterdir())/1e6,2))
if __name__=='__main__':main()