jelson.rodrigues/piperun-bot
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

style: apply cargo fmt

Browse Source
This commit is contained in:
Jelson Stoelben Rodrigues
2025-11-03 10:56:00 -03:00
parent 5477f91bb9
commit 14bf25e83a
2 changed files with 535 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

394
src/groupped_repport.rs
View File

@@ -6,9 +6,9 @@ use polars::prelude::buffer::validate_utf8;
use polars::prelude::*;
use reqwest;
use serde::Deserialize;
use walkdir;
use std::time::Duration;
use std::env;
use std::time::Duration;
use walkdir;
use csv;
@@ -29,7 +29,7 @@ struct CsvEvaluation {
DISPONIBILIDADE: u8,
CONHECIMENTO_TÉCNICO: u8,
DIDATISMO: u8,
ID_TALK: String
ID_TALK: String,
}
// impl TryFrom::<csv::DeserializeRecordsIter<&[u8], CsvHeader>> for CsvEvaluation {
@@ -52,7 +52,8 @@ fn main() {
.unwrap_or("11432".to_string())
.parse::<u16>()
.unwrap_or(11432);
let OLLAMA_AI_MODEL_DATA_SANITIZATION = env::var("OLLAMA_AI_MODEL_DATA_SANITIZATION").expect("Missing environment variable OLLAMA_AI_MODEL_DATA_SANITIZATION");
let OLLAMA_AI_MODEL_DATA_SANITIZATION = env::var("OLLAMA_AI_MODEL_DATA_SANITIZATION")
.expect("Missing environment variable OLLAMA_AI_MODEL_DATA_SANITIZATION");
let ip_address = ipaddress::IPAddress::parse(OLLAMA_URL.to_string());
let OLLAMA_SANITIZED_IP = match ip_address {
@@ -70,166 +71,253 @@ fn main() {
let current_date = chrono::Local::now();
let formatted_date = current_date.format("%Y-%m-%d").to_string();
let previous_week_folder_names = std::fs::read_dir(std::path::Path::new("./evaluations")).expect("Failed to read directory ./evaluations")
.filter_map_ok(|entry| {
if entry.metadata().unwrap().is_dir(){
Some(entry.file_name())
}
else {
None
}
})
.filter_map_ok(|entry_string_name| {
let regex_match_date = regex::Regex::new(r"(\d{4}-\d{2}-\d{2})").expect("Failed to build regex");
let filename = entry_string_name.to_str().unwrap();
let matches_find = regex_match_date.find(filename);
let previous_week_folder_names = std::fs::read_dir(std::path::Path::new("./evaluations"))
.expect("Failed to read directory ./evaluations")
.filter_map_ok(|entry| {
if entry.metadata().unwrap().is_dir() {
Some(entry.file_name())
} else {
None
}
})
.filter_map_ok(|entry_string_name| {
let regex_match_date =
regex::Regex::new(r"(\d{4}-\d{2}-\d{2})").expect("Failed to build regex");
match matches_find {
Some(found) => {
let date = chrono::NaiveDate::parse_from_str(found.as_str(), "%Y-%m-%d");
return Some((date.unwrap().week(chrono::Weekday::Sun), entry_string_name));
},
None => {return None;}
};
})
.filter_map_ok(|(week, directory_string)| {
let current_date = chrono::Local::now();
let first_day_of_current_week = current_date.date_naive().week(chrono::Weekday::Sun).first_day();
let current_date_minus_one_week = first_day_of_current_week.checked_sub_days(chrono::Days::new(1)).expect("Failed to subtract one day");
let first_day_of_last_week = current_date_minus_one_week.week(chrono::Weekday::Sun).first_day();
let first_day_of_week_in_folder_name = week.first_day();
let filename = entry_string_name.to_str().unwrap();
let matches_find = regex_match_date.find(filename);
if first_day_of_last_week == first_day_of_week_in_folder_name {
return Some(directory_string);
}
return None;
})
.filter_map(|value| {
if value.is_ok() {return Some(value.unwrap());}
else {return None;}
})
.sorted()
.collect_vec();
match matches_find {
Some(found) => {
let date = chrono::NaiveDate::parse_from_str(found.as_str(), "%Y-%m-%d");
return Some((date.unwrap().week(chrono::Weekday::Sun), entry_string_name));
}
None => {
return None;
}
};
})
.filter_map_ok(|(week, directory_string)| {
let current_date = chrono::Local::now();
let first_day_of_current_week = current_date
.date_naive()
.week(chrono::Weekday::Sun)
.first_day();
let current_date_minus_one_week = first_day_of_current_week
.checked_sub_days(chrono::Days::new(1))
.expect("Failed to subtract one day");
let first_day_of_last_week = current_date_minus_one_week
.week(chrono::Weekday::Sun)
.first_day();
let first_day_of_week_in_folder_name = week.first_day();
if first_day_of_last_week == first_day_of_week_in_folder_name {
return Some(directory_string);
}
return None;
})
.filter_map(|value| {
if value.is_ok() {
return Some(value.unwrap());
} else {
return None;
}
})
.sorted()
.collect_vec();
println!("{:?}", previous_week_folder_names);
let prompt_data_sanitization = std::fs::read_to_string("./PROMPT_DATA_SANITIZATION.txt").expect("Failed to read PROMPT_DATA_SANITIZATION.txt");
let prompt_data_sanitization = std::fs::read_to_string("./PROMPT_DATA_SANITIZATION.txt")
.expect("Failed to read PROMPT_DATA_SANITIZATION.txt");
let client = reqwest::blocking::Client::new();
let a = previous_week_folder_names.iter().map(|folder_name| {
let a = previous_week_folder_names
.iter()
.map(|folder_name| {
let folder_base_path = std::path::Path::new("./evaluations");
let folder_date_path = folder_base_path.join(folder_name);
std::fs::read_dir(folder_date_path)
})
.filter_map_ok(|files_inside_folder_on_date |
{
files_inside_folder_on_date.filter_ok(|entry| {
let entry_file_name_as_str = entry.file_name().into_string().expect("Failed to get filename as a String");
.take(1)
.filter_map_ok(|files_inside_folder_on_date| {
let groupped_by_user_on_day = files_inside_folder_on_date
.filter_ok(|entry| {
let entry_file_name_as_str = entry
.file_name()
.into_string()
.expect("Failed to get filename as a String");
entry_file_name_as_str.ends_with(".csv") &&
!entry_file_name_as_str.contains("response_time.csv")
})
.filter_map(|value|{
if value.is_ok() {return Some(value.unwrap());}
None
})
.take(1)
.map(|file_name_csv| {
println!("{:?}", file_name_csv.path());
let file_contents = std::fs::read_to_string(file_name_csv.path()).expect("Failed to read CSV file");
let ollama_api_request = client.post(format!("http://{OLLAMA_SANITIZED_IP}:{OLLAMA_PORT}/api/generate"))
.body(
serde_json::json!({
"model": OLLAMA_AI_MODEL_DATA_SANITIZATION,
"prompt": format!("{prompt_data_sanitization} \n{file_contents}"),
"temperature": 0.0, // Get predictable and reproducible output
"stream": false,
}).to_string()
);
let result = ollama_api_request.timeout(Duration::from_secs(3600)).send();
match result {
Ok(response) => {println!("Response: {:?}", response);
let response_json = response.json::<serde_json::Value>().expect("Failed to deserialize response to JSON");
let ai_response = response_json["response"]
.as_str()
.expect("Failed to get AI response as string");
let ai_response = ai_response.to_string();
let ai_response = if let Some(resp) = ai_response.strip_prefix(" ").unwrap_or(&ai_response).strip_prefix("```csv\n") { resp.to_string() } else { ai_response };
let ai_response = if let Some(resp) = ai_response.strip_suffix(" ").unwrap_or(&ai_response).strip_suffix("```") { resp.to_string() } else { ai_response };
return Ok((ai_response, file_name_csv));
},
Err(error) => {println!("Error {error}"); return Err(error);}
};
})
.filter_map_ok(|(ai_repsonse, file_path_csv)| {
let mut reader = csv::ReaderBuilder::new()
.has_headers(true)
.delimiter(b';')
.from_reader(ai_repsonse.as_bytes());
let mut deserialized_iter = reader.deserialize::<CsvHeader>();
let mut columns = deserialized_iter.filter_ok(|value| {
value.PONTOS.is_some()
entry_file_name_as_str.ends_with(".csv")
&& !entry_file_name_as_str.contains("response_time.csv")
})
.map_ok(|value| {
let col = Column::new(value.CATEGORIA.into(), [value.PONTOS.unwrap() as u32]);
col
})
.filter_map(|value|{
if value.is_ok() {return Some(value.unwrap());}
.filter_map(|value| {
if value.is_ok() {
return Some(value.unwrap());
}
None
})
// .take(1)
.map(|file_name_csv| {
println!("{:?}", file_name_csv.path());
let file_contents = std::fs::read_to_string(file_name_csv.path())
.expect("Failed to read CSV file");
let ollama_api_request = client
.post(format!(
"http://{OLLAMA_SANITIZED_IP}:{OLLAMA_PORT}/api/generate"
))
.body(
serde_json::json!({
"model": OLLAMA_AI_MODEL_DATA_SANITIZATION,
"prompt": format!("{prompt_data_sanitization} \n{file_contents}"),
"temperature": 0.0, // Get predictable and reproducible output
"stream": false,
})
.to_string(),
);
let result = ollama_api_request.timeout(Duration::from_secs(3600)).send();
match result {
Ok(response) => {
println!("Response: {:?}", response);
let response_json = response
.json::<serde_json::Value>()
.expect("Failed to deserialize response to JSON");
let ai_response = response_json["response"]
.as_str()
.expect("Failed to get AI response as string");
let ai_response = ai_response.to_string();
let ai_response = if let Some(resp) = ai_response
.strip_prefix(" ")
.unwrap_or(&ai_response)
.strip_prefix("```csv\n")
{
resp.to_string()
} else {
ai_response
};
let ai_response = if let Some(resp) = ai_response
.strip_suffix(" ")
.unwrap_or(&ai_response)
.strip_suffix("```")
{
resp.to_string()
} else {
ai_response
};
return Ok((ai_response, file_name_csv));
}
Err(error) => {
println!("Error {error}");
return Err(error);
}
};
})
.filter_map_ok(|(ai_repsonse, file_path_csv)| {
let mut reader = csv::ReaderBuilder::new()
.has_headers(true)
.delimiter(b';')
.from_reader(ai_repsonse.as_bytes());
let mut deserialized_iter = reader.deserialize::<CsvHeader>();
let mut columns = deserialized_iter
.filter_ok(|value| value.PONTOS.is_some())
.map_ok(|value| {
let col =
Column::new(value.CATEGORIA.into(), [value.PONTOS.unwrap() as u32]);
col
})
.filter_map(|value| {
if value.is_ok() {
return Some(value.unwrap());
}
None
})
.collect_vec();
if columns.len() != 9 {
return None;
}
// Parse id talk from file_path
// filename example is: CC - Erraoander Quintana - 515578 - 20251020515578.csv
// id talk is the last information, so in the example is: 20251020515578
let regex_filename =
regex::Regex::new(r"(CC - )((\w+\s*)+) - (\d+) - (\d+).csv").unwrap();
let filename = file_path_csv
.file_name()
.into_string()
.expect("Failed to convert file name as Rust &str");
let found_regex_groups_in_filename = regex_filename
.captures(filename.as_str())
.expect("Failed to do regex capture");
let user_name = found_regex_groups_in_filename
.get(2)
.expect("Failed to get the id from regex maches");
let talk_id = found_regex_groups_in_filename
.get(5)
.expect("Failed to get the id from regex maches");
columns.push(Column::new("ID_TALK".into(), [talk_id.clone().as_str()]));
let df = polars::frame::DataFrame::new(columns)
.expect("Failed to concatenate into a dataframe");
// println!("{:?}", df);
// Create a dataframe with the evaluation columns plus the talk id
// return a tuple with the dataframe and the user name, so it can be correctly merged after
return Some((user_name.as_str().to_owned(), df));
})
.filter_map(|res| {
if res.is_ok() {
return Some(res.unwrap());
}
return None;
})
.into_group_map()
.into_iter()
.map(|(name, eval_dataframe_vec)| {
let groupped_df = eval_dataframe_vec
.iter()
.cloned()
.reduce(|acc, e| acc.vstack(&e).unwrap())
.expect("Failed to concatenate dataframes");
(name, groupped_df)
})
.collect_vec();
if columns.len() != 9 { return None;}
// Parse id talk from file_path
// filename example is: CC - Erraoander Quintana - 515578 - 20251020515578.csv
// id talk is the last information, so in the example is: 20251020515578
let regex_filename = regex::Regex::new(r"(CC - )((\w+\s*)+) - (\d+) - (\d+).csv").unwrap();
let filename = file_path_csv
.file_name()
.into_string()
.expect("Failed to convert file name as Rust &str");
let found_regex_groups_in_filename = regex_filename.captures(
filename.as_str()
).expect("Failed to do regex capture");
let user_name = found_regex_groups_in_filename.get(2).expect("Failed to get the id from regex maches");
let talk_id = found_regex_groups_in_filename.get(5).expect("Failed to get the id from regex maches");
columns.push(Column::new("ID_TALK".into(), [talk_id.clone().as_str()]));
let df = polars::frame::DataFrame::new(columns).expect("Failed to concatenate into a dataframe");
println!("{:?}", df);
// Create a dataframe with the evaluation columns plus the talk id
// return a tuple with the dataframe and the user name, so it can be correctly merged after
return Some((user_name.as_str().to_owned(), df));
})
.for_each(|username| {});
// println!("{:?}", files_inside_folder_on_date);
return Some(());
return Some(groupped_by_user_on_day);
})
.filter_map(|res| {
if res.is_ok() {
return Some(res.unwrap());
}
return None;
})
.into_group_map()
.into_iter()
.map(|(name, eval_dataframe_vec)| {
let groupped_df = eval_dataframe_vec
.iter()
.cloned()
.reduce(|acc, e| acc.vstack(&e).unwrap())
.expect("Failed to concatenate dataframes");
(name, groupped_df)
})
// .collect_vec();
.collect_vec();
// println!("{:?}", a);
println!("{:?}", a);
// Read CSV files inside folder
@@ -238,14 +326,12 @@ fn main() {
// Save into a hashmap, with the user name as key, the date, evaluation
// Final file should look like
/*
Header: Att1, att2, att3, ...
categoria1
categoria2
categoria3
...
/*
Header: Att1, att2, att3, ...
categoria1
categoria2
categoria3
...
*/
}
*/
}