Modded system to use dates and search for filenames by default.
This commit is contained in:
parent
9396e2da3a
commit
a0da416f88
|
@ -9,6 +9,7 @@
|
||||||
/arxiv-processor/arxiv-processor
|
/arxiv-processor/arxiv-processor
|
||||||
/json2md/json2md
|
/json2md/json2md
|
||||||
/llm_processor/llm_processor
|
/llm_processor/llm_processor
|
||||||
|
paper-system
|
||||||
|
|
||||||
# Dependency directories
|
# Dependency directories
|
||||||
vendor/
|
vendor/
|
||||||
|
@ -27,6 +28,8 @@ temp_*.json
|
||||||
.env
|
.env
|
||||||
|
|
||||||
# Generated output files
|
# Generated output files
|
||||||
|
*.md
|
||||||
|
*.json
|
||||||
papers.json
|
papers.json
|
||||||
newpapers.md.gz
|
newpapers.md.gz
|
||||||
arxiv-processor/newpapers.json
|
arxiv-processor/newpapers.json
|
||||||
|
|
36
main.go
36
main.go
|
@ -169,10 +169,11 @@ func main() {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to marshal papers: %v", err)
|
log.Fatalf("Failed to marshal papers: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile("papers.json", papersJSON, 0644); err != nil {
|
jsonName := fmt.Sprintf("%s-%s-%s-papers.json", *start, *end, *search)
|
||||||
|
if err := os.WriteFile(jsonName, papersJSON, 0644); err != nil {
|
||||||
log.Fatalf("Failed to save papers JSON: %v", err)
|
log.Fatalf("Failed to save papers JSON: %v", err)
|
||||||
}
|
}
|
||||||
log.Printf("Successfully saved papers to papers.json")
|
log.Printf("Successfully saved papers to %s", jsonName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Print paper titles for verification
|
// Print paper titles for verification
|
||||||
|
@ -182,12 +183,17 @@ func main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save papers to temporary file for LLM processing
|
// Save papers to temporary file for LLM processing
|
||||||
tempInput := "temp_input.json"
|
tempInput, err := os.CreateTemp("", "arxiv-process-*.tmp")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to create temp file: %v", err)
|
||||||
|
}
|
||||||
|
defer os.Remove(tempInput.Name())
|
||||||
|
|
||||||
tempJSON, err := json.Marshal(llmPapers)
|
tempJSON, err := json.Marshal(llmPapers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to marshal papers for LLM: %v", err)
|
log.Fatalf("Failed to marshal papers for LLM: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(tempInput, tempJSON, 0644); err != nil {
|
if err := os.WriteFile(tempInput.Name(), tempJSON, 0644); err != nil {
|
||||||
log.Fatalf("Failed to save temp input JSON: %v", err)
|
log.Fatalf("Failed to save temp input JSON: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -201,30 +207,38 @@ func main() {
|
||||||
llmProcessor := processor.NewProcessor(*model, 32, apiKey) // 32 = batch size from README
|
llmProcessor := processor.NewProcessor(*model, 32, apiKey) // 32 = batch size from README
|
||||||
log.Printf("Initialized LLM processor with model %s", *model)
|
log.Printf("Initialized LLM processor with model %s", *model)
|
||||||
|
|
||||||
tempOutput := "temp_output.json"
|
tempOutput, err := os.CreateTemp("", "arxiv-process-*.tmp")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to create temp file: %v", err)
|
||||||
|
}
|
||||||
|
defer os.Remove(tempOutput.Name())
|
||||||
|
|
||||||
log.Printf("Processing papers with criteria from %s", *criteriaFile)
|
log.Printf("Processing papers with criteria from %s", *criteriaFile)
|
||||||
if err := llmProcessor.ProcessPapers(ctx, tempInput, tempOutput, *criteriaFile, 1*time.Second); err != nil {
|
if err := llmProcessor.ProcessPapers(ctx, tempInput.Name(), tempOutput.Name(), *criteriaFile, 1*time.Second); err != nil {
|
||||||
log.Fatalf("LLM processing failed: %v", err)
|
log.Fatalf("LLM processing failed: %v", err)
|
||||||
}
|
}
|
||||||
log.Printf("LLM processing complete, results saved to %s", tempOutput)
|
log.Printf("LLM processing complete, results saved to %s", tempOutput)
|
||||||
|
|
||||||
// Generate markdown
|
// Generate markdown
|
||||||
log.Printf("Generating markdown output")
|
log.Printf("Generating markdown output")
|
||||||
decisions, err := lib.ProcessJSONFile(tempOutput)
|
decisions, err := lib.ProcessJSONFile(tempOutput.Name())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to process JSON: %v", err)
|
log.Fatalf("Failed to process JSON: %v", err)
|
||||||
}
|
}
|
||||||
log.Printf("Processed decisions: %d accepted, %d rejected", len(decisions.Accepted), len(decisions.Rejected))
|
log.Printf("Processed decisions: %d accepted, %d rejected", len(decisions.Accepted), len(decisions.Rejected))
|
||||||
|
|
||||||
|
defaultOutput := fmt.Sprintf("%s-%s-%s-papers.md", *start, *end, *search)
|
||||||
|
if *output == "papers.md" && usingArxiv {
|
||||||
|
*output = defaultOutput
|
||||||
|
}
|
||||||
|
|
||||||
if err := lib.GenerateMarkdown(decisions, *output); err != nil {
|
if err := lib.GenerateMarkdown(decisions, *output); err != nil {
|
||||||
log.Fatalf("Markdown generation failed: %v", err)
|
log.Fatalf("Markdown generation failed: %v", err)
|
||||||
}
|
}
|
||||||
log.Printf("Generated markdown output at %s", *output)
|
log.Printf("Generated markdown output at %s", *output)
|
||||||
|
|
||||||
// Cleanup temp files
|
// Temp files cleaned up by defer statements
|
||||||
os.Remove(tempInput)
|
log.Printf("Cleanup complete")
|
||||||
os.Remove(tempOutput)
|
|
||||||
log.Printf("Cleaned up temporary files")
|
|
||||||
|
|
||||||
log.Printf("Process complete. Results saved to %s", *output)
|
log.Printf("Process complete. Results saved to %s", *output)
|
||||||
}
|
}
|
||||||
|
|
BIN
paper-system
BIN
paper-system
Binary file not shown.
Loading…
Reference in New Issue