Armored decisions more; smaller models have more variance

paperprocessor.go

@@ -92,7 +92,7 @@ func (p *Processor) ProcessPapers(papers []Paper, criteria string) (*ProcessingR
 			}{
 				Paper:  paper,
 				Error:  err.Error(),
-				Output: "", // We could potentially add the raw LLM output here if needed
+				Output: decision.RawOutput, // Include raw output for debugging
 			})
 			continue
 		}
@@ -142,6 +142,7 @@ type llmResponse struct {
 type decisionResult struct {
 	Decision    string
 	Explanation string
+	RawOutput   string // Store raw output for error reporting
 }
 
 func (p *Processor) evaluatePaper(paper Paper, criteria string) (*decisionResult, error) {
@@ -171,12 +172,12 @@ Abstract: %s`, criteria, paper.Title, paper.Abstract)
 
 	reqJSON, err := json.Marshal(reqBody)
 	if err != nil {
-		return nil, fmt.Errorf("error marshaling request: %v", err)
+		return &decisionResult{RawOutput: ""}, fmt.Errorf("error marshaling request: %v", err)
 	}
 
 	req, err := http.NewRequest("POST", p.config.APIEndpoint, bytes.NewBuffer(reqJSON))
 	if err != nil {
-		return nil, fmt.Errorf("error creating request: %v", err)
+		return &decisionResult{RawOutput: ""}, fmt.Errorf("error creating request: %v", err)
 	}
 
 	req.Header.Set("Content-Type", "application/json")
@@ -185,39 +186,48 @@ Abstract: %s`, criteria, paper.Title, paper.Abstract)
 	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("error making request: %v", err)
+		return &decisionResult{RawOutput: ""}, fmt.Errorf("error making request: %v", err)
 	}
 	defer resp.Body.Close()
 
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("error reading response: %v", err)
+		return &decisionResult{RawOutput: ""}, fmt.Errorf("error reading response: %v", err)
 	}
 
 	var llmResp llmResponse
 	if err := json.Unmarshal(body, &llmResp); err != nil {
-		return nil, fmt.Errorf("error unmarshaling response: %v", err)
+		return &decisionResult{RawOutput: string(body)}, fmt.Errorf("error unmarshaling response: %v", err)
 	}
 
 	if len(llmResp.Choices) == 0 {
-		return nil, fmt.Errorf("no response from LLM")
+		return &decisionResult{RawOutput: string(body)}, fmt.Errorf("no response from LLM")
 	}
 
 	content := llmResp.Choices[0].Message.Content
 
-	// Find first line with ACCEPT/REJECT
+	// Find line with ACCEPT/REJECT
 	var decisionLine string
 	lines := bytes.Split([]byte(content), []byte("\n"))
-	for _, line := range lines {
-		if strings.Contains(strings.ToUpper(string(line)), "ACCEPT") ||
-			strings.Contains(strings.ToUpper(string(line)), "REJECT") {
+	for i, line := range lines {
+		upperLine := strings.ToUpper(string(line))
+		// Check current line
+		if strings.Contains(upperLine, "ACCEPT") || strings.Contains(upperLine, "REJECT") {
 			decisionLine = string(line)
 			break
 		}
+		// If current line is "DECISION", check next line
+		if strings.TrimSpace(upperLine) == "DECISION" && i+1 < len(lines) {
+			nextLine := strings.ToUpper(string(lines[i+1]))
+			if strings.Contains(nextLine, "ACCEPT") || strings.Contains(nextLine, "REJECT") {
+				decisionLine = string(lines[i+1])
+				break
+			}
+		}
 	}
 
 	if decisionLine == "" {
-		return nil, fmt.Errorf("no decision found in response. Full response:\n%s", content)
+		return &decisionResult{RawOutput: content}, fmt.Errorf("no decision found in response. Full response:\n%s", content)
 	}
 
 	// Clean and normalize decision
@@ -225,7 +235,7 @@ Abstract: %s`, criteria, paper.Title, paper.Abstract)
 
 	// Handle common prefixes and clean the decision text
 	cleanDecision := rawDecision
-	for _, prefix := range []string{"DECISION:", "Decision:", "-", "\"", "*"} {
+	for _, prefix := range []string{"DECISION:", "Decision:", "-", "\"", "*", "THIS PAPER IS"} {
 		cleanDecision = strings.TrimPrefix(cleanDecision, prefix)
 	}
 	cleanDecision = strings.TrimSpace(cleanDecision)
@@ -241,16 +251,25 @@ Abstract: %s`, criteria, paper.Title, paper.Abstract)
 	case strings.HasPrefix(upperDecision, "REJECT"):
 		decision = "REJECT"
 	default:
-		return nil, fmt.Errorf("invalid decision value: %q (cleaned: %q). Full response:\n%s",
+		return &decisionResult{RawOutput: content}, fmt.Errorf("invalid decision value: %q (cleaned: %q). Full response:\n%s",
 			rawDecision, cleanDecision, content)
 	}
 
 	// Get explanation as everything after the decision line
 	explanation := strings.TrimSpace(strings.Replace(content, decisionLine, "", 1))
 
+	// Remove any "Explanation" header if present
+	explanation = strings.TrimPrefix(strings.TrimSpace(explanation), "Explanation")
+	explanation = strings.TrimSpace(explanation)
+
+	if explanation == "" {
+		return &decisionResult{RawOutput: content}, fmt.Errorf("empty explanation in response. Full response:\n%s", content)
+	}
+
 	return &decisionResult{
 		Decision:    decision,
 		Explanation: explanation,
+		RawOutput:   content,
 	}, nil
 }