Modded system to use dates and search for filenames by default.
This commit is contained in:
parent
9396e2da3a
commit
a0da416f88
|
@ -9,6 +9,7 @@
|
|||
/arxiv-processor/arxiv-processor
|
||||
/json2md/json2md
|
||||
/llm_processor/llm_processor
|
||||
paper-system
|
||||
|
||||
# Dependency directories
|
||||
vendor/
|
||||
|
@ -27,6 +28,8 @@ temp_*.json
|
|||
.env
|
||||
|
||||
# Generated output files
|
||||
*.md
|
||||
*.json
|
||||
papers.json
|
||||
newpapers.md.gz
|
||||
arxiv-processor/newpapers.json
|
||||
|
|
36
main.go
36
main.go
|
@ -169,10 +169,11 @@ func main() {
|
|||
if err != nil {
|
||||
log.Fatalf("Failed to marshal papers: %v", err)
|
||||
}
|
||||
if err := os.WriteFile("papers.json", papersJSON, 0644); err != nil {
|
||||
jsonName := fmt.Sprintf("%s-%s-%s-papers.json", *start, *end, *search)
|
||||
if err := os.WriteFile(jsonName, papersJSON, 0644); err != nil {
|
||||
log.Fatalf("Failed to save papers JSON: %v", err)
|
||||
}
|
||||
log.Printf("Successfully saved papers to papers.json")
|
||||
log.Printf("Successfully saved papers to %s", jsonName)
|
||||
}
|
||||
|
||||
// Print paper titles for verification
|
||||
|
@ -182,12 +183,17 @@ func main() {
|
|||
}
|
||||
|
||||
// Save papers to temporary file for LLM processing
|
||||
tempInput := "temp_input.json"
|
||||
tempInput, err := os.CreateTemp("", "arxiv-process-*.tmp")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create temp file: %v", err)
|
||||
}
|
||||
defer os.Remove(tempInput.Name())
|
||||
|
||||
tempJSON, err := json.Marshal(llmPapers)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to marshal papers for LLM: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(tempInput, tempJSON, 0644); err != nil {
|
||||
if err := os.WriteFile(tempInput.Name(), tempJSON, 0644); err != nil {
|
||||
log.Fatalf("Failed to save temp input JSON: %v", err)
|
||||
}
|
||||
|
||||
|
@ -201,30 +207,38 @@ func main() {
|
|||
llmProcessor := processor.NewProcessor(*model, 32, apiKey) // 32 = batch size from README
|
||||
log.Printf("Initialized LLM processor with model %s", *model)
|
||||
|
||||
tempOutput := "temp_output.json"
|
||||
tempOutput, err := os.CreateTemp("", "arxiv-process-*.tmp")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create temp file: %v", err)
|
||||
}
|
||||
defer os.Remove(tempOutput.Name())
|
||||
|
||||
log.Printf("Processing papers with criteria from %s", *criteriaFile)
|
||||
if err := llmProcessor.ProcessPapers(ctx, tempInput, tempOutput, *criteriaFile, 1*time.Second); err != nil {
|
||||
if err := llmProcessor.ProcessPapers(ctx, tempInput.Name(), tempOutput.Name(), *criteriaFile, 1*time.Second); err != nil {
|
||||
log.Fatalf("LLM processing failed: %v", err)
|
||||
}
|
||||
log.Printf("LLM processing complete, results saved to %s", tempOutput)
|
||||
|
||||
// Generate markdown
|
||||
log.Printf("Generating markdown output")
|
||||
decisions, err := lib.ProcessJSONFile(tempOutput)
|
||||
decisions, err := lib.ProcessJSONFile(tempOutput.Name())
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to process JSON: %v", err)
|
||||
}
|
||||
log.Printf("Processed decisions: %d accepted, %d rejected", len(decisions.Accepted), len(decisions.Rejected))
|
||||
|
||||
defaultOutput := fmt.Sprintf("%s-%s-%s-papers.md", *start, *end, *search)
|
||||
if *output == "papers.md" && usingArxiv {
|
||||
*output = defaultOutput
|
||||
}
|
||||
|
||||
if err := lib.GenerateMarkdown(decisions, *output); err != nil {
|
||||
log.Fatalf("Markdown generation failed: %v", err)
|
||||
}
|
||||
log.Printf("Generated markdown output at %s", *output)
|
||||
|
||||
// Cleanup temp files
|
||||
os.Remove(tempInput)
|
||||
os.Remove(tempOutput)
|
||||
log.Printf("Cleaned up temporary files")
|
||||
// Temp files cleaned up by defer statements
|
||||
log.Printf("Cleanup complete")
|
||||
|
||||
log.Printf("Process complete. Results saved to %s", *output)
|
||||
}
|
||||
|
|
BIN
paper-system
BIN
paper-system
Binary file not shown.
Loading…
Reference in New Issue