Compare commits
5 Commits
80d60acf64
...
0d44120a63
Author | SHA1 | Date |
---|---|---|
Steve White | 0d44120a63 | |
Steve White | 171698a56b | |
Steve White | 698e7caa6f | |
Steve White | 4c2adc636b | |
Steve White | 36fdd5f00e |
21
README.md
21
README.md
|
@ -16,6 +16,23 @@ Go package for automated evaluation of academic papers using LLM-based criteria
|
||||||
go get gitea.r8z.us/stwhite/paperprocessor
|
go get gitea.r8z.us/stwhite/paperprocessor
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## API Reference
|
||||||
|
|
||||||
|
### ProcessFile
|
||||||
|
`func ProcessFile(inputPath, outputPath, criteriaPath string, config Config, debug bool) error`
|
||||||
|
|
||||||
|
Processes papers from input JSON file and writes results to output JSON file
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
- inputPath: Path to input JSON file containing papers array
|
||||||
|
- outputPath: Path to write processing results JSON
|
||||||
|
- criteriaPath: Path to text file with evaluation criteria
|
||||||
|
- config: Configuration settings for API and processing
|
||||||
|
- debug: Enable debug logging when true
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
- error: Processing error or nil if successful
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
### Basic Configuration
|
### Basic Configuration
|
||||||
|
@ -37,7 +54,11 @@ err := paperprocessor.ProcessFile(
|
||||||
"output/results.json",
|
"output/results.json",
|
||||||
"criteria.txt",
|
"criteria.txt",
|
||||||
config,
|
config,
|
||||||
|
true, // debug mode
|
||||||
)
|
)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Processing failed:", err)
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## Input Formats
|
## Input Formats
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Paper represents a single academic paper
|
// Paper represents a single academic paper
|
||||||
|
@ -123,7 +124,8 @@ type decisionResult struct {
|
||||||
|
|
||||||
func (p *Processor) evaluatePaper(paper Paper, criteria string) (*decisionResult, error) {
|
func (p *Processor) evaluatePaper(paper Paper, criteria string) (*decisionResult, error) {
|
||||||
prompt := fmt.Sprintf(`Please evaluate the following academic paper against the provided criteria.
|
prompt := fmt.Sprintf(`Please evaluate the following academic paper against the provided criteria.
|
||||||
Respond with either "ACCEPT" or "REJECT" followed by a brief explanation of your decision.
|
Respond with either "ACCEPT" or "REJECT" followed by a brief explanation of your decision.
|
||||||
|
Do not use markdown emphasis in your response. Keep your response clear and concise.
|
||||||
Your response should be in the format:
|
Your response should be in the format:
|
||||||
DECISION
|
DECISION
|
||||||
Explanation
|
Explanation
|
||||||
|
@ -179,22 +181,46 @@ Abstract: %s`, criteria, paper.Title, paper.Abstract)
|
||||||
}
|
}
|
||||||
|
|
||||||
content := llmResp.Choices[0].Message.Content
|
content := llmResp.Choices[0].Message.Content
|
||||||
|
|
||||||
|
// Find first line with ACCEPT/REJECT
|
||||||
|
var decisionLine string
|
||||||
lines := bytes.Split([]byte(content), []byte("\n"))
|
lines := bytes.Split([]byte(content), []byte("\n"))
|
||||||
|
for _, line := range lines {
|
||||||
if len(lines) < 2 {
|
if strings.Contains(strings.ToUpper(string(line)), "ACCEPT") ||
|
||||||
return nil, fmt.Errorf("invalid response format")
|
strings.Contains(strings.ToUpper(string(line)), "REJECT") {
|
||||||
|
decisionLine = string(line)
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
decisionParts := bytes.SplitN(lines[0], []byte(":"), 2)
|
if decisionLine == "" {
|
||||||
if len(decisionParts) != 2 {
|
return nil, fmt.Errorf("no decision found in response. Full response:\n%s", content)
|
||||||
return nil, fmt.Errorf("invalid decision format, expected 'DECISION: [VALUE]'")
|
|
||||||
}
|
|
||||||
decision := strings.ToUpper(string(bytes.TrimSpace(decisionParts[1])))
|
|
||||||
if decision != "ACCEPT" && decision != "REJECT" {
|
|
||||||
return nil, fmt.Errorf("invalid decision value: %s", decision)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
explanation := string(bytes.TrimSpace(bytes.Join(lines[1:], []byte("\n"))))
|
// Clean and normalize decision
|
||||||
|
rawDecision := strings.TrimSpace(decisionLine)
|
||||||
|
|
||||||
|
// Remove "DECISION:" prefix if present and trim non-alphabetic characters
|
||||||
|
cleanDecision := strings.TrimPrefix(rawDecision, "DECISION:")
|
||||||
|
cleanDecision = strings.TrimFunc(cleanDecision, func(r rune) bool {
|
||||||
|
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Normalize case and check for valid decision
|
||||||
|
upperDecision := strings.ToUpper(cleanDecision)
|
||||||
|
var decision string
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(upperDecision, "ACCEPT"):
|
||||||
|
decision = "ACCEPT"
|
||||||
|
case strings.HasPrefix(upperDecision, "REJECT"):
|
||||||
|
decision = "REJECT"
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("invalid decision value: %q (cleaned: %q). Full response:\n%s",
|
||||||
|
rawDecision, cleanDecision, content)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get explanation as everything after the decision line
|
||||||
|
explanation := strings.TrimSpace(strings.Replace(content, decisionLine, "", 1))
|
||||||
|
|
||||||
return &decisionResult{
|
return &decisionResult{
|
||||||
Decision: decision,
|
Decision: decision,
|
||||||
|
|
Loading…
Reference in New Issue