christopher
/
rsslair
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Refactor the sqlite database to only ever have one connection 

- Only allow one sqlite connection by using a static sqlite connection
- Remove the Database() constructor from lua. Access the database from
  the global db object
This commit is contained in:
Christopher Williams 2024-10-07 17:44:48 -04:00
parent ca4a157505
commit 35a2cd0606
7 changed files with 145 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/9to5mac.lua
View File

@ -9,8 +9,6 @@ function Nineto5mac.route(args)
local feed = rss_parser:new(xml)
local articles = feed.channel.articles
local db = Database()
local existing_articles = db:check_if_articles_in_feed_exist(feed)
log:debug("Fetching missing articles from the database")

2
scripts/godot.lua
View File

@ -9,8 +9,6 @@ function godot.route(args)
local feed = rss_parser:new(xml)
local articles = feed.channel.articles
local db = Database()
local existing_articles = db:check_if_articles_in_feed_exist(feed)
log:debug("Fetching missing articles from the database")

1
scripts/phoronix.lua
View File

@ -13,7 +13,6 @@ function phoronix.route(args)
local articles = feed.channel.articles -- Get all of the article objects
-- TODO: Add api to check if the articles are already in the database
local db = Database()
local existing_articles = db:check_if_articles_in_feed_exist(feed) -- Get the missing articles from the database
log:debug("Fetching missing articles from the database")

13
scripts/techcrunch.lua
View File

@ -13,7 +13,6 @@ function techCrunch.route(args)
local articles = feed.channel.articles -- Get all of the article objects
-- TODO: Add api to check if the articles are already in the database
local db = Database()
local existing_articles = db:check_if_articles_in_feed_exist(feed) -- Get the missing articles from the database
log:debug("Fetching missing articles from the database")
@ -26,15 +25,15 @@ function techCrunch.route(args)
log:debug("Getting article: " .. article.title .. " from " .. article.link)
local article_content = get(article.link) -- Get the entire article content
local html_parser = HtmlParser() -- Create a new instance of the html parser
html_parser:parse(article_content) -- Parse the article into an html tree
local article_content = get(article.link) -- Get the entire article content
local html_parser = HtmlParser() -- Create a new instance of the html parser
html_parser:parse(article_content) -- Parse the article into an html tree
local elements = html_parser:select_element('.wp-block-post-content') -- Select the element with the class 'wp-block-post-content'
local elements = html_parser:select_element('.wp-block-post-content') -- Select the element with the class 'wp-block-post-content'
local element = elements
[1] -- String of the html from the element selected
[1] -- String of the html from the element selected
article.description =
element -- Replace the description with the entire article
element -- Replace the description with the entire article
sleep(500)
::continue::
end

149
src/database.rs
View File

@ -1,19 +1,28 @@
use std::{collections::HashMap, rc::Rc};
use crate::rss_parser::{Item as Article, Rss};
use log::{debug, error, trace};
use mlua::{FromLua, Lua, Table, UserData, UserDataMethods, Value};
use once_cell::sync::Lazy;
use rusqlite::{params, Connection};
use crate::{implement_from_lua, rss_parser::{Item as Article, Rss}};
use std::sync::Mutex;
static SQLITE_DB: Lazy<Mutex<Connection>> =
Lazy::new(|| Mutex::new(Connection::open("database.db").unwrap()));
#[derive(Debug, Clone)]
pub struct Database {
conn: Rc<Connection>,
conn: &'static Mutex<Connection>,
}
impl Default for Database {
fn default() -> Self {
Self::new()
}
}
impl Database {
pub fn new() -> Database {
//TODO: Handle all of these unwrap calls
let conn = Connection::open("database.db").unwrap();
let conn = SQLITE_DB.lock().unwrap();
conn.execute(
"CREATE TABLE IF NOT EXISTS feeds (
id INTEGER PRIMARY KEY,
@ -22,7 +31,8 @@ impl Database {
last_updated TEXT NOT NULL
)",
params![],
).unwrap();
)
.unwrap();
conn.execute(
"CREATE TABLE IF NOT EXISTS images (
url TEXT PRIMARY KEY,
@ -32,7 +42,8 @@ impl Database {
height INTEGER NOT NULL
)",
params![],
).unwrap();
)
.unwrap();
conn.execute(
"CREATE TABLE IF NOT EXISTS articles (
id INTEGER PRIMARY KEY,
@ -46,13 +57,9 @@ impl Database {
description TEXT NOT NULL
)",
params![],
).unwrap();
//Ok(Self {
// conn: Rc::new(conn),
//})
Self {
conn: Rc::new(conn),
}
)
.unwrap();
Self { conn: &*SQLITE_DB }
}
pub fn new_with_conn(conn: Connection) -> Database {
@ -64,7 +71,8 @@ impl Database {
last_updated TEXT NOT NULL
)",
params![],
).unwrap();
)
.unwrap();
conn.execute(
"CREATE TABLE IF NOT EXISTS images (
url TEXT PRIMARY KEY,
@ -74,7 +82,8 @@ impl Database {
height INTEGER NOT NULL
)",
params![],
).unwrap();
)
.unwrap();
conn.execute(
"CREATE TABLE IF NOT EXISTS articles (
id INTEGER PRIMARY KEY,
@ -88,14 +97,19 @@ impl Database {
description TEXT NOT NULL
)",
params![],
).unwrap();
Self {
conn: Rc::new(conn),
}
)
.unwrap();
Self { conn: &*SQLITE_DB }
}
pub fn insert_feed(&self, url: &str, title: &str, last_updated: &str) -> Result<usize, Box<dyn std::error::Error>> {
Ok(self.conn.execute(
pub fn insert_feed(
&self,
url: &str,
title: &str,
last_updated: &str,
) -> Result<usize, Box<dyn std::error::Error>> {
let conn = self.conn.lock().unwrap();
Ok(conn.execute(
"INSERT INTO feeds (url, title, last_updated) VALUES (?1, ?2, ?3)",
params![url, title, last_updated],
)?)
@ -110,14 +124,12 @@ impl Database {
/// # Returns
///
/// * The number of rows inserted
pub fn insert_article(
&self,
article: &Article
) -> Result<usize, Box<dyn std::error::Error>> {
pub fn insert_article(&self, article: &Article) -> Result<usize, Box<dyn std::error::Error>> {
let title = article.title.as_ref().unwrap();
let guid = article.guid.as_ref().unwrap().value.clone();
let ret = self.conn.execute(
let conn = self.conn.lock().unwrap();
let ret = conn.execute(
"INSERT INTO articles (guid, creator, categories, feed_id, title, link, pub_date, description) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![
guid,
@ -144,26 +156,27 @@ impl Database {
pub fn get_article_by_guid(&self, guid: &str) -> mlua::Result<Option<Article>> {
//TODO: Make this look better
let mut stmt = match self.conn.prepare("SELECT * FROM articles WHERE guid = ?1") {
let conn = self.conn.lock().unwrap();
let mut stmt = match conn.prepare("SELECT * FROM articles WHERE guid = ?1") {
Ok(it) => it,
Err(err) => {
let var_name = Err(err);
return var_name.unwrap();
},
}
};
let mut rows = match stmt.query(params![guid]) {
Ok(it) => it,
Err(err) => {
let var_name = Err(err);
return var_name.unwrap();
},
}
};
let row = match rows.next() {
Ok(it) => it,
Err(err) => {
let var_name = Err(err);
return var_name.unwrap();
},
}
};
let row = match row {
Some(it) => it,
@ -177,7 +190,12 @@ impl Database {
is_perma_link: Some(false),
}),
creator: Some(row.get(2).unwrap_or("".to_string())),
categories: row.get(3).unwrap_or("".to_string()).split(",").map(|s| s.to_string()).collect(),
categories: row
.get(3)
.unwrap_or("".to_string())
.split(",")
.map(|s| s.to_string())
.collect(),
link: Some(row.get(6).unwrap_or("".to_string())),
pub_date: Some(row.get(7).unwrap_or("".to_string())),
description: Some(row.get(8).unwrap_or("".to_string())),
@ -216,42 +234,47 @@ impl FromLua<'_> for Database {
impl UserData for Database {
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
methods.add_method("get_article_by_guid", |_, this, guid: String| -> Result<Article, mlua::Error> {
match this.get_article_by_guid(&guid) {
Ok(Some(article)) => Ok(article),
Ok(None) => Err(mlua::Error::RuntimeError("Article not found".to_string())),
Err(e) => Err(mlua::Error::RuntimeError(e.to_string())),
}
});
methods.add_method("check_if_articles_in_feed_exist", |lua, this, feed: Rss| -> Result<Table, mlua::Error> {
// Check to if all of the articles in the feed exist
let table = lua.create_table()?;
let articles = &feed.channel.borrow().items;
for article in articles {
let article = article.borrow();
let guid = &article.guid.as_ref().unwrap().value;
let guid = guid.clone().unwrap();
methods.add_method(
"get_article_by_guid",
|_, this, guid: String| -> Result<Article, mlua::Error> {
match this.get_article_by_guid(&guid) {
Ok(Some(article)) => {
table.set(guid, article.description.clone().unwrap())?;
},
Ok(None) => {
continue;
},
Err(e) => {
error!("Error checking if article exists: {}", e);
},
Ok(Some(article)) => Ok(article),
Ok(None) => Err(mlua::Error::RuntimeError("Article not found".to_string())),
Err(e) => Err(mlua::Error::RuntimeError(e.to_string())),
}
}
},
);
Ok(table)
});
methods.add_method(
"check_if_articles_in_feed_exist",
|lua, this, feed: Rss| -> Result<Table, mlua::Error> {
// Check to if all of the articles in the feed exist
let table = lua.create_table()?;
let articles = &feed.channel.borrow().items;
for article in articles {
let article = article.borrow();
let guid = &article.guid.as_ref().unwrap().value;
let guid = guid.clone().unwrap();
match this.get_article_by_guid(&guid) {
Ok(Some(article)) => {
table.set(guid, article.description.clone().unwrap())?;
}
Ok(None) => {
continue;
}
Err(e) => {
error!("Error checking if article exists: {}", e);
}
}
}
Ok(table)
},
);
//methods.add_method("does_article_exist", |_, this, guid: String| -> Result<Result<bool, _>, mlua::Error> {
// Ok(this.does_article_exist(&guid))
//});
}
}

19
src/scheduler.rs
View File

@ -3,33 +3,22 @@ use chrono::prelude::*;
use chrono::Utc;
use log::debug;
use log::error;
use log::trace;
use redis::Commands;
use std::str::FromStr;
use std::thread;
use crate::database::Database;
use crate::scripting::ScriptingEngine;
use crate::REDIS;
pub struct Scheduler {
engine: ScriptingEngine,
db: Database,
}
impl Scheduler {
pub fn new(scripting: ScriptingEngine) -> Result<Self> {
Ok(Self {
engine: scripting,
db: Database::new(),
})
Ok(Self { engine: scripting })
}
pub fn new_with_db(scripting: ScriptingEngine, db: Database) -> Self {
Self {
engine: scripting,
db,
}
pub fn new_with_db(scripting: ScriptingEngine) -> Self {
Self { engine: scripting }
}
/// Example of a cron expression every minute: `0/1 * * * *`
@ -46,7 +35,7 @@ impl Scheduler {
// TODO: Check if scheduling 1 minute after now to prevent scheduling the same minute again
let until = next - now + chrono::Duration::minutes(1);
let sleep_time = match until.to_std() {
Ok(duration) => duration,
Err(_) => {

78
src/scripting.rs
View File

@ -1,12 +1,12 @@
use crate::database::Database;
use crate::{html_parser::HtmlParser, rss_parser::AtomLink};
use crate::rss_parser::{Channel, Item, Rss};
use crate::REDIS;
use crate::{html_parser::HtmlParser, rss_parser::AtomLink};
use redis::Commands;
use log::{debug, error, info, trace, warn};
use mlua::{chunk, AsChunk, Chunk, ExternalResult, Function, Lua, MetaMethod, Result, Table, UserData, UserDataMethods};
use mlua::{
chunk, ExternalResult, Function, Lua, MetaMethod, Result, Table, UserData, UserDataMethods
};
use std::collections::HashMap;
pub struct ScriptingEngine {
@ -77,7 +77,9 @@ impl ScriptingEngine {
pub fn load_globals(&self) -> Result<()> {
macro_rules! add_constructor {
($name:literal, $constructor:ident) => {
let constructor = self.lua.create_function(|_, ()| Ok($constructor::default()))?;
let constructor = self
.lua
.create_function(|_, ()| Ok($constructor::default()))?;
self.lua.globals().set($name, constructor)?;
};
}
@ -94,7 +96,7 @@ impl ScriptingEngine {
self.lua.globals().set($name, $value)?;
};
}
add_global!("log", Log);
//TODO: This is an async function... may be able to be used, but not on a tokio thread
//add_global!("get", self.lua.create_async_function(|_, url: String| async move {
@ -108,27 +110,35 @@ impl ScriptingEngine {
// Ok(response.text().await.unwrap())
//})?);
add_global!("get", self.lua.create_function(|_, url: String| {
debug!("Making a GET request to {}", url);
let response = reqwest::blocking::get(&url)
.and_then(|response| response.error_for_status())
.into_lua_err()
.unwrap();
debug!("Response: {:?}", response);
Ok(response.text().unwrap())
})?);
add_global!("sleep", self.lua.create_function(|_, milliseconds: u64| {
debug!("Sleeping for {} milliseconds", milliseconds);
std::thread::sleep(std::time::Duration::from_millis(milliseconds));
Ok(())
})?);
add_global!(
"get",
self.lua.create_function(|_, url: String| {
debug!("Making a GET request to {}", url);
let response = reqwest::blocking::get(&url)
.and_then(|response| response.error_for_status())
.into_lua_err()
.unwrap();
debug!("Response: {:?}", response);
Ok(response.text().unwrap())
})?
);
add_global!(
"sleep",
self.lua.create_function(|_, milliseconds: u64| {
debug!("Sleeping for {} milliseconds", milliseconds);
std::thread::sleep(std::time::Duration::from_millis(milliseconds));
Ok(())
})?
);
add_constructor!("HtmlParser", HtmlParser);
add_constructor!("Feed", Rss);
add_constructor!("Channel", Channel);
add_constructor!("AtomLink", AtomLink);
add_constructor!("Article", Item);
add_constructor_new!("Database", Database);
let database = Database::new();
self.lua.globals().set("db", database)?;
self.lua.globals().set("routes", self.lua.create_table()?)?;
let add_route = chunk! {
@ -145,7 +155,7 @@ impl ScriptingEngine {
Ok(())
}
/// Execute all of the routes that are registered
/// Execute all of the routes that are registered
pub fn execute_all_routes(&mut self) -> Result<()> {
let routes = self.get_routes().unwrap();
debug!("Executing all lua script routes");
@ -161,7 +171,7 @@ impl ScriptingEngine {
continue;
}
};
let result = ret.0;
let feed = ret.1;
@ -182,7 +192,7 @@ impl ScriptingEngine {
let article = article.borrow();
match self.db.insert_article(&article) {
Ok(_) => (),
Err(_) => continue
Err(_) => continue,
}
}
debug!("Finished inserting articles into database");
@ -196,7 +206,11 @@ impl ScriptingEngine {
/// # Arguments
///
/// * `route_name` - The name of the route to execute
pub fn execute_route_function(&self, obj_name: &str, route_name: &str) -> Result<(String, Rss)> {
pub fn execute_route_function(
&self,
obj_name: &str,
route_name: &str,
) -> Result<(String, Rss)> {
let route_obj: Table = match self.lua.globals().get(obj_name) {
Ok(route) => route,
Err(_) => {
@ -226,7 +240,10 @@ impl ScriptingEngine {
}
pub fn get_routes(&self) -> Result<HashMap<String, String>> {
let routes = self.lua.globals().get::<_, HashMap<String, String>>("routes")?;
let routes = self
.lua
.globals()
.get::<_, HashMap<String, String>>("routes")?;
Ok(routes)
}
@ -301,8 +318,8 @@ mod tests {
};
let ret = db.insert_article(&article);
assert!(ret.is_ok());
let script = chunk!{
let script = chunk! {
local db = $db;
local article = db:get_article_by_guid("https://techcrunch.com/?p=2877786");
return article;
@ -312,7 +329,10 @@ mod tests {
let ret = engine.lua.load(script).eval::<Article>()?;
assert_eq!(ret.title.unwrap(), "Test");
assert_eq!(ret.guid.unwrap().value.unwrap(), "https://techcrunch.com/?p=2877786");
assert_eq!(
ret.guid.unwrap().value.unwrap(),
"https://techcrunch.com/?p=2877786"
);
assert_eq!(ret.link.unwrap(), "https://example.com");
assert_eq!(ret.pub_date.unwrap(), "2021-01-01T00:00:00Z");
assert_eq!(ret.description.unwrap(), "Test");