Compare commits
No commits in common. "main" and "golang" have entirely different histories.
|
|
@ -1,5 +0,0 @@
|
||||||
target/
|
|
||||||
test/
|
|
||||||
*.db
|
|
||||||
*.txt
|
|
||||||
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
/target
|
*.db
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
36
Cargo.toml
36
Cargo.toml
|
|
@ -1,36 +0,0 @@
|
||||||
[package]
|
|
||||||
name = "rsslair"
|
|
||||||
version = "0.1.0"
|
|
||||||
edition = "2021"
|
|
||||||
|
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
anyhow = "1.0.87"
|
|
||||||
chrono = "0.4.38"
|
|
||||||
clap = { version = "4.5.18", features = ["derive"] }
|
|
||||||
clap_derive = "4.5.18"
|
|
||||||
cron = "0.12.1"
|
|
||||||
cron-lingo = "0.4.2"
|
|
||||||
env_logger = "0.11.5"
|
|
||||||
html5ever = "0.29.0"
|
|
||||||
hyper = "1.4.1"
|
|
||||||
hyper-router = "0.5.0"
|
|
||||||
log = "0.4.22"
|
|
||||||
markup5ever = "0.14.0"
|
|
||||||
markup5ever_rcdom = "0.3.0"
|
|
||||||
minidom = "0.16.0"
|
|
||||||
mlua = { version = "0.9.9", features = ["lua54", "macros", "async"] }
|
|
||||||
once_cell = "1.19.0"
|
|
||||||
quick-xml = { version = "0.36.1", features = ["serialize"]}
|
|
||||||
redis = { version = "0.27.2", features = ["tokio-comp", "aio"] }
|
|
||||||
reqwest = { version = "0.12.7", features = ["blocking"] }
|
|
||||||
rusqlite = "0.32.1"
|
|
||||||
scraper = "0.20.0"
|
|
||||||
serde = { version = "1.0.130", features = ["derive", "rc"]}
|
|
||||||
serde_json = "1.0.128"
|
|
||||||
tiny_http = "0.12.0"
|
|
||||||
tokio = { version = "1.40.0", features = ["full"] }
|
|
||||||
tokio-macros = "2.4.0"
|
|
||||||
warp = "0.3.7"
|
|
||||||
xml = "0.8.20"
|
|
||||||
20
Dockerfile
20
Dockerfile
|
|
@ -1,17 +1,11 @@
|
||||||
FROM rust as builder
|
FROM golang:1.21.5
|
||||||
|
|
||||||
|
COPY . /app
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY . .
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install lua5.4 liblua5.4-dev -y
|
RUN go mod download
|
||||||
RUN cargo build --release
|
RUN go build -o rsslair .
|
||||||
|
|
||||||
|
# Set the entrypoint
|
||||||
FROM debian:bookworm-slim as runtime
|
ENTRYPOINT ["/app/rsslair"]
|
||||||
WORKDIR /app
|
|
||||||
COPY --from=builder /app/target/release/rsslair .
|
|
||||||
COPY ./run.bash .
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install liblua5.4-0 libssl3 sqlite3 -y
|
|
||||||
|
|
||||||
CMD ["./run.bash"]
|
|
||||||
|
|
||||||
|
|
|
||||||
661
LICENSE
661
LICENSE
|
|
@ -1,661 +0,0 @@
|
||||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
|
||||||
Version 3, 19 November 2007
|
|
||||||
|
|
||||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
|
||||||
Everyone is permitted to copy and distribute verbatim copies
|
|
||||||
of this license document, but changing it is not allowed.
|
|
||||||
|
|
||||||
Preamble
|
|
||||||
|
|
||||||
The GNU Affero General Public License is a free, copyleft license for
|
|
||||||
software and other kinds of works, specifically designed to ensure
|
|
||||||
cooperation with the community in the case of network server software.
|
|
||||||
|
|
||||||
The licenses for most software and other practical works are designed
|
|
||||||
to take away your freedom to share and change the works. By contrast,
|
|
||||||
our General Public Licenses are intended to guarantee your freedom to
|
|
||||||
share and change all versions of a program--to make sure it remains free
|
|
||||||
software for all its users.
|
|
||||||
|
|
||||||
When we speak of free software, we are referring to freedom, not
|
|
||||||
price. Our General Public Licenses are designed to make sure that you
|
|
||||||
have the freedom to distribute copies of free software (and charge for
|
|
||||||
them if you wish), that you receive source code or can get it if you
|
|
||||||
want it, that you can change the software or use pieces of it in new
|
|
||||||
free programs, and that you know you can do these things.
|
|
||||||
|
|
||||||
Developers that use our General Public Licenses protect your rights
|
|
||||||
with two steps: (1) assert copyright on the software, and (2) offer
|
|
||||||
you this License which gives you legal permission to copy, distribute
|
|
||||||
and/or modify the software.
|
|
||||||
|
|
||||||
A secondary benefit of defending all users' freedom is that
|
|
||||||
improvements made in alternate versions of the program, if they
|
|
||||||
receive widespread use, become available for other developers to
|
|
||||||
incorporate. Many developers of free software are heartened and
|
|
||||||
encouraged by the resulting cooperation. However, in the case of
|
|
||||||
software used on network servers, this result may fail to come about.
|
|
||||||
The GNU General Public License permits making a modified version and
|
|
||||||
letting the public access it on a server without ever releasing its
|
|
||||||
source code to the public.
|
|
||||||
|
|
||||||
The GNU Affero General Public License is designed specifically to
|
|
||||||
ensure that, in such cases, the modified source code becomes available
|
|
||||||
to the community. It requires the operator of a network server to
|
|
||||||
provide the source code of the modified version running there to the
|
|
||||||
users of that server. Therefore, public use of a modified version, on
|
|
||||||
a publicly accessible server, gives the public access to the source
|
|
||||||
code of the modified version.
|
|
||||||
|
|
||||||
An older license, called the Affero General Public License and
|
|
||||||
published by Affero, was designed to accomplish similar goals. This is
|
|
||||||
a different license, not a version of the Affero GPL, but Affero has
|
|
||||||
released a new version of the Affero GPL which permits relicensing under
|
|
||||||
this license.
|
|
||||||
|
|
||||||
The precise terms and conditions for copying, distribution and
|
|
||||||
modification follow.
|
|
||||||
|
|
||||||
TERMS AND CONDITIONS
|
|
||||||
|
|
||||||
0. Definitions.
|
|
||||||
|
|
||||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
|
||||||
|
|
||||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
|
||||||
works, such as semiconductor masks.
|
|
||||||
|
|
||||||
"The Program" refers to any copyrightable work licensed under this
|
|
||||||
License. Each licensee is addressed as "you". "Licensees" and
|
|
||||||
"recipients" may be individuals or organizations.
|
|
||||||
|
|
||||||
To "modify" a work means to copy from or adapt all or part of the work
|
|
||||||
in a fashion requiring copyright permission, other than the making of an
|
|
||||||
exact copy. The resulting work is called a "modified version" of the
|
|
||||||
earlier work or a work "based on" the earlier work.
|
|
||||||
|
|
||||||
A "covered work" means either the unmodified Program or a work based
|
|
||||||
on the Program.
|
|
||||||
|
|
||||||
To "propagate" a work means to do anything with it that, without
|
|
||||||
permission, would make you directly or secondarily liable for
|
|
||||||
infringement under applicable copyright law, except executing it on a
|
|
||||||
computer or modifying a private copy. Propagation includes copying,
|
|
||||||
distribution (with or without modification), making available to the
|
|
||||||
public, and in some countries other activities as well.
|
|
||||||
|
|
||||||
To "convey" a work means any kind of propagation that enables other
|
|
||||||
parties to make or receive copies. Mere interaction with a user through
|
|
||||||
a computer network, with no transfer of a copy, is not conveying.
|
|
||||||
|
|
||||||
An interactive user interface displays "Appropriate Legal Notices"
|
|
||||||
to the extent that it includes a convenient and prominently visible
|
|
||||||
feature that (1) displays an appropriate copyright notice, and (2)
|
|
||||||
tells the user that there is no warranty for the work (except to the
|
|
||||||
extent that warranties are provided), that licensees may convey the
|
|
||||||
work under this License, and how to view a copy of this License. If
|
|
||||||
the interface presents a list of user commands or options, such as a
|
|
||||||
menu, a prominent item in the list meets this criterion.
|
|
||||||
|
|
||||||
1. Source Code.
|
|
||||||
|
|
||||||
The "source code" for a work means the preferred form of the work
|
|
||||||
for making modifications to it. "Object code" means any non-source
|
|
||||||
form of a work.
|
|
||||||
|
|
||||||
A "Standard Interface" means an interface that either is an official
|
|
||||||
standard defined by a recognized standards body, or, in the case of
|
|
||||||
interfaces specified for a particular programming language, one that
|
|
||||||
is widely used among developers working in that language.
|
|
||||||
|
|
||||||
The "System Libraries" of an executable work include anything, other
|
|
||||||
than the work as a whole, that (a) is included in the normal form of
|
|
||||||
packaging a Major Component, but which is not part of that Major
|
|
||||||
Component, and (b) serves only to enable use of the work with that
|
|
||||||
Major Component, or to implement a Standard Interface for which an
|
|
||||||
implementation is available to the public in source code form. A
|
|
||||||
"Major Component", in this context, means a major essential component
|
|
||||||
(kernel, window system, and so on) of the specific operating system
|
|
||||||
(if any) on which the executable work runs, or a compiler used to
|
|
||||||
produce the work, or an object code interpreter used to run it.
|
|
||||||
|
|
||||||
The "Corresponding Source" for a work in object code form means all
|
|
||||||
the source code needed to generate, install, and (for an executable
|
|
||||||
work) run the object code and to modify the work, including scripts to
|
|
||||||
control those activities. However, it does not include the work's
|
|
||||||
System Libraries, or general-purpose tools or generally available free
|
|
||||||
programs which are used unmodified in performing those activities but
|
|
||||||
which are not part of the work. For example, Corresponding Source
|
|
||||||
includes interface definition files associated with source files for
|
|
||||||
the work, and the source code for shared libraries and dynamically
|
|
||||||
linked subprograms that the work is specifically designed to require,
|
|
||||||
such as by intimate data communication or control flow between those
|
|
||||||
subprograms and other parts of the work.
|
|
||||||
|
|
||||||
The Corresponding Source need not include anything that users
|
|
||||||
can regenerate automatically from other parts of the Corresponding
|
|
||||||
Source.
|
|
||||||
|
|
||||||
The Corresponding Source for a work in source code form is that
|
|
||||||
same work.
|
|
||||||
|
|
||||||
2. Basic Permissions.
|
|
||||||
|
|
||||||
All rights granted under this License are granted for the term of
|
|
||||||
copyright on the Program, and are irrevocable provided the stated
|
|
||||||
conditions are met. This License explicitly affirms your unlimited
|
|
||||||
permission to run the unmodified Program. The output from running a
|
|
||||||
covered work is covered by this License only if the output, given its
|
|
||||||
content, constitutes a covered work. This License acknowledges your
|
|
||||||
rights of fair use or other equivalent, as provided by copyright law.
|
|
||||||
|
|
||||||
You may make, run and propagate covered works that you do not
|
|
||||||
convey, without conditions so long as your license otherwise remains
|
|
||||||
in force. You may convey covered works to others for the sole purpose
|
|
||||||
of having them make modifications exclusively for you, or provide you
|
|
||||||
with facilities for running those works, provided that you comply with
|
|
||||||
the terms of this License in conveying all material for which you do
|
|
||||||
not control copyright. Those thus making or running the covered works
|
|
||||||
for you must do so exclusively on your behalf, under your direction
|
|
||||||
and control, on terms that prohibit them from making any copies of
|
|
||||||
your copyrighted material outside their relationship with you.
|
|
||||||
|
|
||||||
Conveying under any other circumstances is permitted solely under
|
|
||||||
the conditions stated below. Sublicensing is not allowed; section 10
|
|
||||||
makes it unnecessary.
|
|
||||||
|
|
||||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
|
||||||
|
|
||||||
No covered work shall be deemed part of an effective technological
|
|
||||||
measure under any applicable law fulfilling obligations under article
|
|
||||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
|
||||||
similar laws prohibiting or restricting circumvention of such
|
|
||||||
measures.
|
|
||||||
|
|
||||||
When you convey a covered work, you waive any legal power to forbid
|
|
||||||
circumvention of technological measures to the extent such circumvention
|
|
||||||
is effected by exercising rights under this License with respect to
|
|
||||||
the covered work, and you disclaim any intention to limit operation or
|
|
||||||
modification of the work as a means of enforcing, against the work's
|
|
||||||
users, your or third parties' legal rights to forbid circumvention of
|
|
||||||
technological measures.
|
|
||||||
|
|
||||||
4. Conveying Verbatim Copies.
|
|
||||||
|
|
||||||
You may convey verbatim copies of the Program's source code as you
|
|
||||||
receive it, in any medium, provided that you conspicuously and
|
|
||||||
appropriately publish on each copy an appropriate copyright notice;
|
|
||||||
keep intact all notices stating that this License and any
|
|
||||||
non-permissive terms added in accord with section 7 apply to the code;
|
|
||||||
keep intact all notices of the absence of any warranty; and give all
|
|
||||||
recipients a copy of this License along with the Program.
|
|
||||||
|
|
||||||
You may charge any price or no price for each copy that you convey,
|
|
||||||
and you may offer support or warranty protection for a fee.
|
|
||||||
|
|
||||||
5. Conveying Modified Source Versions.
|
|
||||||
|
|
||||||
You may convey a work based on the Program, or the modifications to
|
|
||||||
produce it from the Program, in the form of source code under the
|
|
||||||
terms of section 4, provided that you also meet all of these conditions:
|
|
||||||
|
|
||||||
a) The work must carry prominent notices stating that you modified
|
|
||||||
it, and giving a relevant date.
|
|
||||||
|
|
||||||
b) The work must carry prominent notices stating that it is
|
|
||||||
released under this License and any conditions added under section
|
|
||||||
7. This requirement modifies the requirement in section 4 to
|
|
||||||
"keep intact all notices".
|
|
||||||
|
|
||||||
c) You must license the entire work, as a whole, under this
|
|
||||||
License to anyone who comes into possession of a copy. This
|
|
||||||
License will therefore apply, along with any applicable section 7
|
|
||||||
additional terms, to the whole of the work, and all its parts,
|
|
||||||
regardless of how they are packaged. This License gives no
|
|
||||||
permission to license the work in any other way, but it does not
|
|
||||||
invalidate such permission if you have separately received it.
|
|
||||||
|
|
||||||
d) If the work has interactive user interfaces, each must display
|
|
||||||
Appropriate Legal Notices; however, if the Program has interactive
|
|
||||||
interfaces that do not display Appropriate Legal Notices, your
|
|
||||||
work need not make them do so.
|
|
||||||
|
|
||||||
A compilation of a covered work with other separate and independent
|
|
||||||
works, which are not by their nature extensions of the covered work,
|
|
||||||
and which are not combined with it such as to form a larger program,
|
|
||||||
in or on a volume of a storage or distribution medium, is called an
|
|
||||||
"aggregate" if the compilation and its resulting copyright are not
|
|
||||||
used to limit the access or legal rights of the compilation's users
|
|
||||||
beyond what the individual works permit. Inclusion of a covered work
|
|
||||||
in an aggregate does not cause this License to apply to the other
|
|
||||||
parts of the aggregate.
|
|
||||||
|
|
||||||
6. Conveying Non-Source Forms.
|
|
||||||
|
|
||||||
You may convey a covered work in object code form under the terms
|
|
||||||
of sections 4 and 5, provided that you also convey the
|
|
||||||
machine-readable Corresponding Source under the terms of this License,
|
|
||||||
in one of these ways:
|
|
||||||
|
|
||||||
a) Convey the object code in, or embodied in, a physical product
|
|
||||||
(including a physical distribution medium), accompanied by the
|
|
||||||
Corresponding Source fixed on a durable physical medium
|
|
||||||
customarily used for software interchange.
|
|
||||||
|
|
||||||
b) Convey the object code in, or embodied in, a physical product
|
|
||||||
(including a physical distribution medium), accompanied by a
|
|
||||||
written offer, valid for at least three years and valid for as
|
|
||||||
long as you offer spare parts or customer support for that product
|
|
||||||
model, to give anyone who possesses the object code either (1) a
|
|
||||||
copy of the Corresponding Source for all the software in the
|
|
||||||
product that is covered by this License, on a durable physical
|
|
||||||
medium customarily used for software interchange, for a price no
|
|
||||||
more than your reasonable cost of physically performing this
|
|
||||||
conveying of source, or (2) access to copy the
|
|
||||||
Corresponding Source from a network server at no charge.
|
|
||||||
|
|
||||||
c) Convey individual copies of the object code with a copy of the
|
|
||||||
written offer to provide the Corresponding Source. This
|
|
||||||
alternative is allowed only occasionally and noncommercially, and
|
|
||||||
only if you received the object code with such an offer, in accord
|
|
||||||
with subsection 6b.
|
|
||||||
|
|
||||||
d) Convey the object code by offering access from a designated
|
|
||||||
place (gratis or for a charge), and offer equivalent access to the
|
|
||||||
Corresponding Source in the same way through the same place at no
|
|
||||||
further charge. You need not require recipients to copy the
|
|
||||||
Corresponding Source along with the object code. If the place to
|
|
||||||
copy the object code is a network server, the Corresponding Source
|
|
||||||
may be on a different server (operated by you or a third party)
|
|
||||||
that supports equivalent copying facilities, provided you maintain
|
|
||||||
clear directions next to the object code saying where to find the
|
|
||||||
Corresponding Source. Regardless of what server hosts the
|
|
||||||
Corresponding Source, you remain obligated to ensure that it is
|
|
||||||
available for as long as needed to satisfy these requirements.
|
|
||||||
|
|
||||||
e) Convey the object code using peer-to-peer transmission, provided
|
|
||||||
you inform other peers where the object code and Corresponding
|
|
||||||
Source of the work are being offered to the general public at no
|
|
||||||
charge under subsection 6d.
|
|
||||||
|
|
||||||
A separable portion of the object code, whose source code is excluded
|
|
||||||
from the Corresponding Source as a System Library, need not be
|
|
||||||
included in conveying the object code work.
|
|
||||||
|
|
||||||
A "User Product" is either (1) a "consumer product", which means any
|
|
||||||
tangible personal property which is normally used for personal, family,
|
|
||||||
or household purposes, or (2) anything designed or sold for incorporation
|
|
||||||
into a dwelling. In determining whether a product is a consumer product,
|
|
||||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
|
||||||
product received by a particular user, "normally used" refers to a
|
|
||||||
typical or common use of that class of product, regardless of the status
|
|
||||||
of the particular user or of the way in which the particular user
|
|
||||||
actually uses, or expects or is expected to use, the product. A product
|
|
||||||
is a consumer product regardless of whether the product has substantial
|
|
||||||
commercial, industrial or non-consumer uses, unless such uses represent
|
|
||||||
the only significant mode of use of the product.
|
|
||||||
|
|
||||||
"Installation Information" for a User Product means any methods,
|
|
||||||
procedures, authorization keys, or other information required to install
|
|
||||||
and execute modified versions of a covered work in that User Product from
|
|
||||||
a modified version of its Corresponding Source. The information must
|
|
||||||
suffice to ensure that the continued functioning of the modified object
|
|
||||||
code is in no case prevented or interfered with solely because
|
|
||||||
modification has been made.
|
|
||||||
|
|
||||||
If you convey an object code work under this section in, or with, or
|
|
||||||
specifically for use in, a User Product, and the conveying occurs as
|
|
||||||
part of a transaction in which the right of possession and use of the
|
|
||||||
User Product is transferred to the recipient in perpetuity or for a
|
|
||||||
fixed term (regardless of how the transaction is characterized), the
|
|
||||||
Corresponding Source conveyed under this section must be accompanied
|
|
||||||
by the Installation Information. But this requirement does not apply
|
|
||||||
if neither you nor any third party retains the ability to install
|
|
||||||
modified object code on the User Product (for example, the work has
|
|
||||||
been installed in ROM).
|
|
||||||
|
|
||||||
The requirement to provide Installation Information does not include a
|
|
||||||
requirement to continue to provide support service, warranty, or updates
|
|
||||||
for a work that has been modified or installed by the recipient, or for
|
|
||||||
the User Product in which it has been modified or installed. Access to a
|
|
||||||
network may be denied when the modification itself materially and
|
|
||||||
adversely affects the operation of the network or violates the rules and
|
|
||||||
protocols for communication across the network.
|
|
||||||
|
|
||||||
Corresponding Source conveyed, and Installation Information provided,
|
|
||||||
in accord with this section must be in a format that is publicly
|
|
||||||
documented (and with an implementation available to the public in
|
|
||||||
source code form), and must require no special password or key for
|
|
||||||
unpacking, reading or copying.
|
|
||||||
|
|
||||||
7. Additional Terms.
|
|
||||||
|
|
||||||
"Additional permissions" are terms that supplement the terms of this
|
|
||||||
License by making exceptions from one or more of its conditions.
|
|
||||||
Additional permissions that are applicable to the entire Program shall
|
|
||||||
be treated as though they were included in this License, to the extent
|
|
||||||
that they are valid under applicable law. If additional permissions
|
|
||||||
apply only to part of the Program, that part may be used separately
|
|
||||||
under those permissions, but the entire Program remains governed by
|
|
||||||
this License without regard to the additional permissions.
|
|
||||||
|
|
||||||
When you convey a copy of a covered work, you may at your option
|
|
||||||
remove any additional permissions from that copy, or from any part of
|
|
||||||
it. (Additional permissions may be written to require their own
|
|
||||||
removal in certain cases when you modify the work.) You may place
|
|
||||||
additional permissions on material, added by you to a covered work,
|
|
||||||
for which you have or can give appropriate copyright permission.
|
|
||||||
|
|
||||||
Notwithstanding any other provision of this License, for material you
|
|
||||||
add to a covered work, you may (if authorized by the copyright holders of
|
|
||||||
that material) supplement the terms of this License with terms:
|
|
||||||
|
|
||||||
a) Disclaiming warranty or limiting liability differently from the
|
|
||||||
terms of sections 15 and 16 of this License; or
|
|
||||||
|
|
||||||
b) Requiring preservation of specified reasonable legal notices or
|
|
||||||
author attributions in that material or in the Appropriate Legal
|
|
||||||
Notices displayed by works containing it; or
|
|
||||||
|
|
||||||
c) Prohibiting misrepresentation of the origin of that material, or
|
|
||||||
requiring that modified versions of such material be marked in
|
|
||||||
reasonable ways as different from the original version; or
|
|
||||||
|
|
||||||
d) Limiting the use for publicity purposes of names of licensors or
|
|
||||||
authors of the material; or
|
|
||||||
|
|
||||||
e) Declining to grant rights under trademark law for use of some
|
|
||||||
trade names, trademarks, or service marks; or
|
|
||||||
|
|
||||||
f) Requiring indemnification of licensors and authors of that
|
|
||||||
material by anyone who conveys the material (or modified versions of
|
|
||||||
it) with contractual assumptions of liability to the recipient, for
|
|
||||||
any liability that these contractual assumptions directly impose on
|
|
||||||
those licensors and authors.
|
|
||||||
|
|
||||||
All other non-permissive additional terms are considered "further
|
|
||||||
restrictions" within the meaning of section 10. If the Program as you
|
|
||||||
received it, or any part of it, contains a notice stating that it is
|
|
||||||
governed by this License along with a term that is a further
|
|
||||||
restriction, you may remove that term. If a license document contains
|
|
||||||
a further restriction but permits relicensing or conveying under this
|
|
||||||
License, you may add to a covered work material governed by the terms
|
|
||||||
of that license document, provided that the further restriction does
|
|
||||||
not survive such relicensing or conveying.
|
|
||||||
|
|
||||||
If you add terms to a covered work in accord with this section, you
|
|
||||||
must place, in the relevant source files, a statement of the
|
|
||||||
additional terms that apply to those files, or a notice indicating
|
|
||||||
where to find the applicable terms.
|
|
||||||
|
|
||||||
Additional terms, permissive or non-permissive, may be stated in the
|
|
||||||
form of a separately written license, or stated as exceptions;
|
|
||||||
the above requirements apply either way.
|
|
||||||
|
|
||||||
8. Termination.
|
|
||||||
|
|
||||||
You may not propagate or modify a covered work except as expressly
|
|
||||||
provided under this License. Any attempt otherwise to propagate or
|
|
||||||
modify it is void, and will automatically terminate your rights under
|
|
||||||
this License (including any patent licenses granted under the third
|
|
||||||
paragraph of section 11).
|
|
||||||
|
|
||||||
However, if you cease all violation of this License, then your
|
|
||||||
license from a particular copyright holder is reinstated (a)
|
|
||||||
provisionally, unless and until the copyright holder explicitly and
|
|
||||||
finally terminates your license, and (b) permanently, if the copyright
|
|
||||||
holder fails to notify you of the violation by some reasonable means
|
|
||||||
prior to 60 days after the cessation.
|
|
||||||
|
|
||||||
Moreover, your license from a particular copyright holder is
|
|
||||||
reinstated permanently if the copyright holder notifies you of the
|
|
||||||
violation by some reasonable means, this is the first time you have
|
|
||||||
received notice of violation of this License (for any work) from that
|
|
||||||
copyright holder, and you cure the violation prior to 30 days after
|
|
||||||
your receipt of the notice.
|
|
||||||
|
|
||||||
Termination of your rights under this section does not terminate the
|
|
||||||
licenses of parties who have received copies or rights from you under
|
|
||||||
this License. If your rights have been terminated and not permanently
|
|
||||||
reinstated, you do not qualify to receive new licenses for the same
|
|
||||||
material under section 10.
|
|
||||||
|
|
||||||
9. Acceptance Not Required for Having Copies.
|
|
||||||
|
|
||||||
You are not required to accept this License in order to receive or
|
|
||||||
run a copy of the Program. Ancillary propagation of a covered work
|
|
||||||
occurring solely as a consequence of using peer-to-peer transmission
|
|
||||||
to receive a copy likewise does not require acceptance. However,
|
|
||||||
nothing other than this License grants you permission to propagate or
|
|
||||||
modify any covered work. These actions infringe copyright if you do
|
|
||||||
not accept this License. Therefore, by modifying or propagating a
|
|
||||||
covered work, you indicate your acceptance of this License to do so.
|
|
||||||
|
|
||||||
10. Automatic Licensing of Downstream Recipients.
|
|
||||||
|
|
||||||
Each time you convey a covered work, the recipient automatically
|
|
||||||
receives a license from the original licensors, to run, modify and
|
|
||||||
propagate that work, subject to this License. You are not responsible
|
|
||||||
for enforcing compliance by third parties with this License.
|
|
||||||
|
|
||||||
An "entity transaction" is a transaction transferring control of an
|
|
||||||
organization, or substantially all assets of one, or subdividing an
|
|
||||||
organization, or merging organizations. If propagation of a covered
|
|
||||||
work results from an entity transaction, each party to that
|
|
||||||
transaction who receives a copy of the work also receives whatever
|
|
||||||
licenses to the work the party's predecessor in interest had or could
|
|
||||||
give under the previous paragraph, plus a right to possession of the
|
|
||||||
Corresponding Source of the work from the predecessor in interest, if
|
|
||||||
the predecessor has it or can get it with reasonable efforts.
|
|
||||||
|
|
||||||
You may not impose any further restrictions on the exercise of the
|
|
||||||
rights granted or affirmed under this License. For example, you may
|
|
||||||
not impose a license fee, royalty, or other charge for exercise of
|
|
||||||
rights granted under this License, and you may not initiate litigation
|
|
||||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
|
||||||
any patent claim is infringed by making, using, selling, offering for
|
|
||||||
sale, or importing the Program or any portion of it.
|
|
||||||
|
|
||||||
11. Patents.
|
|
||||||
|
|
||||||
A "contributor" is a copyright holder who authorizes use under this
|
|
||||||
License of the Program or a work on which the Program is based. The
|
|
||||||
work thus licensed is called the contributor's "contributor version".
|
|
||||||
|
|
||||||
A contributor's "essential patent claims" are all patent claims
|
|
||||||
owned or controlled by the contributor, whether already acquired or
|
|
||||||
hereafter acquired, that would be infringed by some manner, permitted
|
|
||||||
by this License, of making, using, or selling its contributor version,
|
|
||||||
but do not include claims that would be infringed only as a
|
|
||||||
consequence of further modification of the contributor version. For
|
|
||||||
purposes of this definition, "control" includes the right to grant
|
|
||||||
patent sublicenses in a manner consistent with the requirements of
|
|
||||||
this License.
|
|
||||||
|
|
||||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
|
||||||
patent license under the contributor's essential patent claims, to
|
|
||||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
|
||||||
propagate the contents of its contributor version.
|
|
||||||
|
|
||||||
In the following three paragraphs, a "patent license" is any express
|
|
||||||
agreement or commitment, however denominated, not to enforce a patent
|
|
||||||
(such as an express permission to practice a patent or covenant not to
|
|
||||||
sue for patent infringement). To "grant" such a patent license to a
|
|
||||||
party means to make such an agreement or commitment not to enforce a
|
|
||||||
patent against the party.
|
|
||||||
|
|
||||||
If you convey a covered work, knowingly relying on a patent license,
|
|
||||||
and the Corresponding Source of the work is not available for anyone
|
|
||||||
to copy, free of charge and under the terms of this License, through a
|
|
||||||
publicly available network server or other readily accessible means,
|
|
||||||
then you must either (1) cause the Corresponding Source to be so
|
|
||||||
available, or (2) arrange to deprive yourself of the benefit of the
|
|
||||||
patent license for this particular work, or (3) arrange, in a manner
|
|
||||||
consistent with the requirements of this License, to extend the patent
|
|
||||||
license to downstream recipients. "Knowingly relying" means you have
|
|
||||||
actual knowledge that, but for the patent license, your conveying the
|
|
||||||
covered work in a country, or your recipient's use of the covered work
|
|
||||||
in a country, would infringe one or more identifiable patents in that
|
|
||||||
country that you have reason to believe are valid.
|
|
||||||
|
|
||||||
If, pursuant to or in connection with a single transaction or
|
|
||||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
|
||||||
covered work, and grant a patent license to some of the parties
|
|
||||||
receiving the covered work authorizing them to use, propagate, modify
|
|
||||||
or convey a specific copy of the covered work, then the patent license
|
|
||||||
you grant is automatically extended to all recipients of the covered
|
|
||||||
work and works based on it.
|
|
||||||
|
|
||||||
A patent license is "discriminatory" if it does not include within
|
|
||||||
the scope of its coverage, prohibits the exercise of, or is
|
|
||||||
conditioned on the non-exercise of one or more of the rights that are
|
|
||||||
specifically granted under this License. You may not convey a covered
|
|
||||||
work if you are a party to an arrangement with a third party that is
|
|
||||||
in the business of distributing software, under which you make payment
|
|
||||||
to the third party based on the extent of your activity of conveying
|
|
||||||
the work, and under which the third party grants, to any of the
|
|
||||||
parties who would receive the covered work from you, a discriminatory
|
|
||||||
patent license (a) in connection with copies of the covered work
|
|
||||||
conveyed by you (or copies made from those copies), or (b) primarily
|
|
||||||
for and in connection with specific products or compilations that
|
|
||||||
contain the covered work, unless you entered into that arrangement,
|
|
||||||
or that patent license was granted, prior to 28 March 2007.
|
|
||||||
|
|
||||||
Nothing in this License shall be construed as excluding or limiting
|
|
||||||
any implied license or other defenses to infringement that may
|
|
||||||
otherwise be available to you under applicable patent law.
|
|
||||||
|
|
||||||
12. No Surrender of Others' Freedom.
|
|
||||||
|
|
||||||
If conditions are imposed on you (whether by court order, agreement or
|
|
||||||
otherwise) that contradict the conditions of this License, they do not
|
|
||||||
excuse you from the conditions of this License. If you cannot convey a
|
|
||||||
covered work so as to satisfy simultaneously your obligations under this
|
|
||||||
License and any other pertinent obligations, then as a consequence you may
|
|
||||||
not convey it at all. For example, if you agree to terms that obligate you
|
|
||||||
to collect a royalty for further conveying from those to whom you convey
|
|
||||||
the Program, the only way you could satisfy both those terms and this
|
|
||||||
License would be to refrain entirely from conveying the Program.
|
|
||||||
|
|
||||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
|
||||||
|
|
||||||
Notwithstanding any other provision of this License, if you modify the
|
|
||||||
Program, your modified version must prominently offer all users
|
|
||||||
interacting with it remotely through a computer network (if your version
|
|
||||||
supports such interaction) an opportunity to receive the Corresponding
|
|
||||||
Source of your version by providing access to the Corresponding Source
|
|
||||||
from a network server at no charge, through some standard or customary
|
|
||||||
means of facilitating copying of software. This Corresponding Source
|
|
||||||
shall include the Corresponding Source for any work covered by version 3
|
|
||||||
of the GNU General Public License that is incorporated pursuant to the
|
|
||||||
following paragraph.
|
|
||||||
|
|
||||||
Notwithstanding any other provision of this License, you have
|
|
||||||
permission to link or combine any covered work with a work licensed
|
|
||||||
under version 3 of the GNU General Public License into a single
|
|
||||||
combined work, and to convey the resulting work. The terms of this
|
|
||||||
License will continue to apply to the part which is the covered work,
|
|
||||||
but the work with which it is combined will remain governed by version
|
|
||||||
3 of the GNU General Public License.
|
|
||||||
|
|
||||||
14. Revised Versions of this License.
|
|
||||||
|
|
||||||
The Free Software Foundation may publish revised and/or new versions of
|
|
||||||
the GNU Affero General Public License from time to time. Such new versions
|
|
||||||
will be similar in spirit to the present version, but may differ in detail to
|
|
||||||
address new problems or concerns.
|
|
||||||
|
|
||||||
Each version is given a distinguishing version number. If the
|
|
||||||
Program specifies that a certain numbered version of the GNU Affero General
|
|
||||||
Public License "or any later version" applies to it, you have the
|
|
||||||
option of following the terms and conditions either of that numbered
|
|
||||||
version or of any later version published by the Free Software
|
|
||||||
Foundation. If the Program does not specify a version number of the
|
|
||||||
GNU Affero General Public License, you may choose any version ever published
|
|
||||||
by the Free Software Foundation.
|
|
||||||
|
|
||||||
If the Program specifies that a proxy can decide which future
|
|
||||||
versions of the GNU Affero General Public License can be used, that proxy's
|
|
||||||
public statement of acceptance of a version permanently authorizes you
|
|
||||||
to choose that version for the Program.
|
|
||||||
|
|
||||||
Later license versions may give you additional or different
|
|
||||||
permissions. However, no additional obligations are imposed on any
|
|
||||||
author or copyright holder as a result of your choosing to follow a
|
|
||||||
later version.
|
|
||||||
|
|
||||||
15. Disclaimer of Warranty.
|
|
||||||
|
|
||||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
|
||||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
|
||||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
|
||||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
|
||||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
||||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
|
||||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
|
||||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
|
||||||
|
|
||||||
16. Limitation of Liability.
|
|
||||||
|
|
||||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
|
||||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
|
||||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
|
||||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
|
||||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
|
||||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
|
||||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
|
||||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
|
||||||
SUCH DAMAGES.
|
|
||||||
|
|
||||||
17. Interpretation of Sections 15 and 16.
|
|
||||||
|
|
||||||
If the disclaimer of warranty and limitation of liability provided
|
|
||||||
above cannot be given local legal effect according to their terms,
|
|
||||||
reviewing courts shall apply local law that most closely approximates
|
|
||||||
an absolute waiver of all civil liability in connection with the
|
|
||||||
Program, unless a warranty or assumption of liability accompanies a
|
|
||||||
copy of the Program in return for a fee.
|
|
||||||
|
|
||||||
END OF TERMS AND CONDITIONS
|
|
||||||
|
|
||||||
How to Apply These Terms to Your New Programs
|
|
||||||
|
|
||||||
If you develop a new program, and you want it to be of the greatest
|
|
||||||
possible use to the public, the best way to achieve this is to make it
|
|
||||||
free software which everyone can redistribute and change under these terms.
|
|
||||||
|
|
||||||
To do so, attach the following notices to the program. It is safest
|
|
||||||
to attach them to the start of each source file to most effectively
|
|
||||||
state the exclusion of warranty; and each file should have at least
|
|
||||||
the "copyright" line and a pointer to where the full notice is found.
|
|
||||||
|
|
||||||
<one line to give the program's name and a brief idea of what it does.>
|
|
||||||
Copyright (C) <year> <name of author>
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU Affero General Public License as published
|
|
||||||
by the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU Affero General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU Affero General Public License
|
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Also add information on how to contact you by electronic and paper mail.
|
|
||||||
|
|
||||||
If your software can interact with users remotely through a computer
|
|
||||||
network, you should also make sure that it provides a way for users to
|
|
||||||
get its source. For example, if your program is a web application, its
|
|
||||||
interface could display a "Source" link that leads users to an archive
|
|
||||||
of the code. There are many ways you could offer source, and different
|
|
||||||
solutions will be better for different programs; see section 13 for the
|
|
||||||
specific requirements.
|
|
||||||
|
|
||||||
You should also get your employer (if you work as a programmer) or school,
|
|
||||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
|
||||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
|
||||||
<https://www.gnu.org/licenses/>.
|
|
||||||
|
|
@ -0,0 +1,339 @@
|
||||||
|
GNU GENERAL PUBLIC LICENSE
|
||||||
|
Version 2, June 1991
|
||||||
|
|
||||||
|
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
|
||||||
|
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||||
|
Everyone is permitted to copy and distribute verbatim copies
|
||||||
|
of this license document, but changing it is not allowed.
|
||||||
|
|
||||||
|
Preamble
|
||||||
|
|
||||||
|
The licenses for most software are designed to take away your
|
||||||
|
freedom to share and change it. By contrast, the GNU General Public
|
||||||
|
License is intended to guarantee your freedom to share and change free
|
||||||
|
software--to make sure the software is free for all its users. This
|
||||||
|
General Public License applies to most of the Free Software
|
||||||
|
Foundation's software and to any other program whose authors commit to
|
||||||
|
using it. (Some other Free Software Foundation software is covered by
|
||||||
|
the GNU Lesser General Public License instead.) You can apply it to
|
||||||
|
your programs, too.
|
||||||
|
|
||||||
|
When we speak of free software, we are referring to freedom, not
|
||||||
|
price. Our General Public Licenses are designed to make sure that you
|
||||||
|
have the freedom to distribute copies of free software (and charge for
|
||||||
|
this service if you wish), that you receive source code or can get it
|
||||||
|
if you want it, that you can change the software or use pieces of it
|
||||||
|
in new free programs; and that you know you can do these things.
|
||||||
|
|
||||||
|
To protect your rights, we need to make restrictions that forbid
|
||||||
|
anyone to deny you these rights or to ask you to surrender the rights.
|
||||||
|
These restrictions translate to certain responsibilities for you if you
|
||||||
|
distribute copies of the software, or if you modify it.
|
||||||
|
|
||||||
|
For example, if you distribute copies of such a program, whether
|
||||||
|
gratis or for a fee, you must give the recipients all the rights that
|
||||||
|
you have. You must make sure that they, too, receive or can get the
|
||||||
|
source code. And you must show them these terms so they know their
|
||||||
|
rights.
|
||||||
|
|
||||||
|
We protect your rights with two steps: (1) copyright the software, and
|
||||||
|
(2) offer you this license which gives you legal permission to copy,
|
||||||
|
distribute and/or modify the software.
|
||||||
|
|
||||||
|
Also, for each author's protection and ours, we want to make certain
|
||||||
|
that everyone understands that there is no warranty for this free
|
||||||
|
software. If the software is modified by someone else and passed on, we
|
||||||
|
want its recipients to know that what they have is not the original, so
|
||||||
|
that any problems introduced by others will not reflect on the original
|
||||||
|
authors' reputations.
|
||||||
|
|
||||||
|
Finally, any free program is threatened constantly by software
|
||||||
|
patents. We wish to avoid the danger that redistributors of a free
|
||||||
|
program will individually obtain patent licenses, in effect making the
|
||||||
|
program proprietary. To prevent this, we have made it clear that any
|
||||||
|
patent must be licensed for everyone's free use or not licensed at all.
|
||||||
|
|
||||||
|
The precise terms and conditions for copying, distribution and
|
||||||
|
modification follow.
|
||||||
|
|
||||||
|
GNU GENERAL PUBLIC LICENSE
|
||||||
|
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||||
|
|
||||||
|
0. This License applies to any program or other work which contains
|
||||||
|
a notice placed by the copyright holder saying it may be distributed
|
||||||
|
under the terms of this General Public License. The "Program", below,
|
||||||
|
refers to any such program or work, and a "work based on the Program"
|
||||||
|
means either the Program or any derivative work under copyright law:
|
||||||
|
that is to say, a work containing the Program or a portion of it,
|
||||||
|
either verbatim or with modifications and/or translated into another
|
||||||
|
language. (Hereinafter, translation is included without limitation in
|
||||||
|
the term "modification".) Each licensee is addressed as "you".
|
||||||
|
|
||||||
|
Activities other than copying, distribution and modification are not
|
||||||
|
covered by this License; they are outside its scope. The act of
|
||||||
|
running the Program is not restricted, and the output from the Program
|
||||||
|
is covered only if its contents constitute a work based on the
|
||||||
|
Program (independent of having been made by running the Program).
|
||||||
|
Whether that is true depends on what the Program does.
|
||||||
|
|
||||||
|
1. You may copy and distribute verbatim copies of the Program's
|
||||||
|
source code as you receive it, in any medium, provided that you
|
||||||
|
conspicuously and appropriately publish on each copy an appropriate
|
||||||
|
copyright notice and disclaimer of warranty; keep intact all the
|
||||||
|
notices that refer to this License and to the absence of any warranty;
|
||||||
|
and give any other recipients of the Program a copy of this License
|
||||||
|
along with the Program.
|
||||||
|
|
||||||
|
You may charge a fee for the physical act of transferring a copy, and
|
||||||
|
you may at your option offer warranty protection in exchange for a fee.
|
||||||
|
|
||||||
|
2. You may modify your copy or copies of the Program or any portion
|
||||||
|
of it, thus forming a work based on the Program, and copy and
|
||||||
|
distribute such modifications or work under the terms of Section 1
|
||||||
|
above, provided that you also meet all of these conditions:
|
||||||
|
|
||||||
|
a) You must cause the modified files to carry prominent notices
|
||||||
|
stating that you changed the files and the date of any change.
|
||||||
|
|
||||||
|
b) You must cause any work that you distribute or publish, that in
|
||||||
|
whole or in part contains or is derived from the Program or any
|
||||||
|
part thereof, to be licensed as a whole at no charge to all third
|
||||||
|
parties under the terms of this License.
|
||||||
|
|
||||||
|
c) If the modified program normally reads commands interactively
|
||||||
|
when run, you must cause it, when started running for such
|
||||||
|
interactive use in the most ordinary way, to print or display an
|
||||||
|
announcement including an appropriate copyright notice and a
|
||||||
|
notice that there is no warranty (or else, saying that you provide
|
||||||
|
a warranty) and that users may redistribute the program under
|
||||||
|
these conditions, and telling the user how to view a copy of this
|
||||||
|
License. (Exception: if the Program itself is interactive but
|
||||||
|
does not normally print such an announcement, your work based on
|
||||||
|
the Program is not required to print an announcement.)
|
||||||
|
|
||||||
|
These requirements apply to the modified work as a whole. If
|
||||||
|
identifiable sections of that work are not derived from the Program,
|
||||||
|
and can be reasonably considered independent and separate works in
|
||||||
|
themselves, then this License, and its terms, do not apply to those
|
||||||
|
sections when you distribute them as separate works. But when you
|
||||||
|
distribute the same sections as part of a whole which is a work based
|
||||||
|
on the Program, the distribution of the whole must be on the terms of
|
||||||
|
this License, whose permissions for other licensees extend to the
|
||||||
|
entire whole, and thus to each and every part regardless of who wrote it.
|
||||||
|
|
||||||
|
Thus, it is not the intent of this section to claim rights or contest
|
||||||
|
your rights to work written entirely by you; rather, the intent is to
|
||||||
|
exercise the right to control the distribution of derivative or
|
||||||
|
collective works based on the Program.
|
||||||
|
|
||||||
|
In addition, mere aggregation of another work not based on the Program
|
||||||
|
with the Program (or with a work based on the Program) on a volume of
|
||||||
|
a storage or distribution medium does not bring the other work under
|
||||||
|
the scope of this License.
|
||||||
|
|
||||||
|
3. You may copy and distribute the Program (or a work based on it,
|
||||||
|
under Section 2) in object code or executable form under the terms of
|
||||||
|
Sections 1 and 2 above provided that you also do one of the following:
|
||||||
|
|
||||||
|
a) Accompany it with the complete corresponding machine-readable
|
||||||
|
source code, which must be distributed under the terms of Sections
|
||||||
|
1 and 2 above on a medium customarily used for software interchange; or,
|
||||||
|
|
||||||
|
b) Accompany it with a written offer, valid for at least three
|
||||||
|
years, to give any third party, for a charge no more than your
|
||||||
|
cost of physically performing source distribution, a complete
|
||||||
|
machine-readable copy of the corresponding source code, to be
|
||||||
|
distributed under the terms of Sections 1 and 2 above on a medium
|
||||||
|
customarily used for software interchange; or,
|
||||||
|
|
||||||
|
c) Accompany it with the information you received as to the offer
|
||||||
|
to distribute corresponding source code. (This alternative is
|
||||||
|
allowed only for noncommercial distribution and only if you
|
||||||
|
received the program in object code or executable form with such
|
||||||
|
an offer, in accord with Subsection b above.)
|
||||||
|
|
||||||
|
The source code for a work means the preferred form of the work for
|
||||||
|
making modifications to it. For an executable work, complete source
|
||||||
|
code means all the source code for all modules it contains, plus any
|
||||||
|
associated interface definition files, plus the scripts used to
|
||||||
|
control compilation and installation of the executable. However, as a
|
||||||
|
special exception, the source code distributed need not include
|
||||||
|
anything that is normally distributed (in either source or binary
|
||||||
|
form) with the major components (compiler, kernel, and so on) of the
|
||||||
|
operating system on which the executable runs, unless that component
|
||||||
|
itself accompanies the executable.
|
||||||
|
|
||||||
|
If distribution of executable or object code is made by offering
|
||||||
|
access to copy from a designated place, then offering equivalent
|
||||||
|
access to copy the source code from the same place counts as
|
||||||
|
distribution of the source code, even though third parties are not
|
||||||
|
compelled to copy the source along with the object code.
|
||||||
|
|
||||||
|
4. You may not copy, modify, sublicense, or distribute the Program
|
||||||
|
except as expressly provided under this License. Any attempt
|
||||||
|
otherwise to copy, modify, sublicense or distribute the Program is
|
||||||
|
void, and will automatically terminate your rights under this License.
|
||||||
|
However, parties who have received copies, or rights, from you under
|
||||||
|
this License will not have their licenses terminated so long as such
|
||||||
|
parties remain in full compliance.
|
||||||
|
|
||||||
|
5. You are not required to accept this License, since you have not
|
||||||
|
signed it. However, nothing else grants you permission to modify or
|
||||||
|
distribute the Program or its derivative works. These actions are
|
||||||
|
prohibited by law if you do not accept this License. Therefore, by
|
||||||
|
modifying or distributing the Program (or any work based on the
|
||||||
|
Program), you indicate your acceptance of this License to do so, and
|
||||||
|
all its terms and conditions for copying, distributing or modifying
|
||||||
|
the Program or works based on it.
|
||||||
|
|
||||||
|
6. Each time you redistribute the Program (or any work based on the
|
||||||
|
Program), the recipient automatically receives a license from the
|
||||||
|
original licensor to copy, distribute or modify the Program subject to
|
||||||
|
these terms and conditions. You may not impose any further
|
||||||
|
restrictions on the recipients' exercise of the rights granted herein.
|
||||||
|
You are not responsible for enforcing compliance by third parties to
|
||||||
|
this License.
|
||||||
|
|
||||||
|
7. If, as a consequence of a court judgment or allegation of patent
|
||||||
|
infringement or for any other reason (not limited to patent issues),
|
||||||
|
conditions are imposed on you (whether by court order, agreement or
|
||||||
|
otherwise) that contradict the conditions of this License, they do not
|
||||||
|
excuse you from the conditions of this License. If you cannot
|
||||||
|
distribute so as to satisfy simultaneously your obligations under this
|
||||||
|
License and any other pertinent obligations, then as a consequence you
|
||||||
|
may not distribute the Program at all. For example, if a patent
|
||||||
|
license would not permit royalty-free redistribution of the Program by
|
||||||
|
all those who receive copies directly or indirectly through you, then
|
||||||
|
the only way you could satisfy both it and this License would be to
|
||||||
|
refrain entirely from distribution of the Program.
|
||||||
|
|
||||||
|
If any portion of this section is held invalid or unenforceable under
|
||||||
|
any particular circumstance, the balance of the section is intended to
|
||||||
|
apply and the section as a whole is intended to apply in other
|
||||||
|
circumstances.
|
||||||
|
|
||||||
|
It is not the purpose of this section to induce you to infringe any
|
||||||
|
patents or other property right claims or to contest validity of any
|
||||||
|
such claims; this section has the sole purpose of protecting the
|
||||||
|
integrity of the free software distribution system, which is
|
||||||
|
implemented by public license practices. Many people have made
|
||||||
|
generous contributions to the wide range of software distributed
|
||||||
|
through that system in reliance on consistent application of that
|
||||||
|
system; it is up to the author/donor to decide if he or she is willing
|
||||||
|
to distribute software through any other system and a licensee cannot
|
||||||
|
impose that choice.
|
||||||
|
|
||||||
|
This section is intended to make thoroughly clear what is believed to
|
||||||
|
be a consequence of the rest of this License.
|
||||||
|
|
||||||
|
8. If the distribution and/or use of the Program is restricted in
|
||||||
|
certain countries either by patents or by copyrighted interfaces, the
|
||||||
|
original copyright holder who places the Program under this License
|
||||||
|
may add an explicit geographical distribution limitation excluding
|
||||||
|
those countries, so that distribution is permitted only in or among
|
||||||
|
countries not thus excluded. In such case, this License incorporates
|
||||||
|
the limitation as if written in the body of this License.
|
||||||
|
|
||||||
|
9. The Free Software Foundation may publish revised and/or new versions
|
||||||
|
of the General Public License from time to time. Such new versions will
|
||||||
|
be similar in spirit to the present version, but may differ in detail to
|
||||||
|
address new problems or concerns.
|
||||||
|
|
||||||
|
Each version is given a distinguishing version number. If the Program
|
||||||
|
specifies a version number of this License which applies to it and "any
|
||||||
|
later version", you have the option of following the terms and conditions
|
||||||
|
either of that version or of any later version published by the Free
|
||||||
|
Software Foundation. If the Program does not specify a version number of
|
||||||
|
this License, you may choose any version ever published by the Free Software
|
||||||
|
Foundation.
|
||||||
|
|
||||||
|
10. If you wish to incorporate parts of the Program into other free
|
||||||
|
programs whose distribution conditions are different, write to the author
|
||||||
|
to ask for permission. For software which is copyrighted by the Free
|
||||||
|
Software Foundation, write to the Free Software Foundation; we sometimes
|
||||||
|
make exceptions for this. Our decision will be guided by the two goals
|
||||||
|
of preserving the free status of all derivatives of our free software and
|
||||||
|
of promoting the sharing and reuse of software generally.
|
||||||
|
|
||||||
|
NO WARRANTY
|
||||||
|
|
||||||
|
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
|
||||||
|
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
|
||||||
|
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
|
||||||
|
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
|
||||||
|
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||||
|
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
|
||||||
|
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
|
||||||
|
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
|
||||||
|
REPAIR OR CORRECTION.
|
||||||
|
|
||||||
|
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||||
|
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
|
||||||
|
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||||
|
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
||||||
|
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
|
||||||
|
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
|
||||||
|
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
|
||||||
|
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
|
||||||
|
POSSIBILITY OF SUCH DAMAGES.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
How to Apply These Terms to Your New Programs
|
||||||
|
|
||||||
|
If you develop a new program, and you want it to be of the greatest
|
||||||
|
possible use to the public, the best way to achieve this is to make it
|
||||||
|
free software which everyone can redistribute and change under these terms.
|
||||||
|
|
||||||
|
To do so, attach the following notices to the program. It is safest
|
||||||
|
to attach them to the start of each source file to most effectively
|
||||||
|
convey the exclusion of warranty; and each file should have at least
|
||||||
|
the "copyright" line and a pointer to where the full notice is found.
|
||||||
|
|
||||||
|
<one line to give the program's name and a brief idea of what it does.>
|
||||||
|
Copyright (C) <year> <name of author>
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation; either version 2 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License along
|
||||||
|
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||||
|
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||||
|
|
||||||
|
Also add information on how to contact you by electronic and paper mail.
|
||||||
|
|
||||||
|
If the program is interactive, make it output a short notice like this
|
||||||
|
when it starts in an interactive mode:
|
||||||
|
|
||||||
|
Gnomovision version 69, Copyright (C) year name of author
|
||||||
|
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||||
|
This is free software, and you are welcome to redistribute it
|
||||||
|
under certain conditions; type `show c' for details.
|
||||||
|
|
||||||
|
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||||
|
parts of the General Public License. Of course, the commands you use may
|
||||||
|
be called something other than `show w' and `show c'; they could even be
|
||||||
|
mouse-clicks or menu items--whatever suits your program.
|
||||||
|
|
||||||
|
You should also get your employer (if you work as a programmer) or your
|
||||||
|
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||||
|
necessary. Here is a sample; alter the names:
|
||||||
|
|
||||||
|
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
|
||||||
|
`Gnomovision' (which makes passes at compilers) written by James Hacker.
|
||||||
|
|
||||||
|
<signature of Ty Coon>, 1 April 1989
|
||||||
|
Ty Coon, President of Vice
|
||||||
|
|
||||||
|
This General Public License does not permit incorporating your program into
|
||||||
|
proprietary programs. If your program is a subroutine library, you may
|
||||||
|
consider it more useful to permit linking proprietary applications with the
|
||||||
|
library. If this is what you want to do, use the GNU Lesser General
|
||||||
|
Public License instead of this License.
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/17xande/configdir"
|
||||||
|
"github.com/pelletier/go-toml"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AppConfig struct {
|
||||||
|
SyncInterval int // Sync time in minutes
|
||||||
|
ShouldCache bool // Should we cache the rss feeds
|
||||||
|
Scripts []string // Scripts to execute
|
||||||
|
}
|
||||||
|
|
||||||
|
func createDefaultConfig(configPath string) {
|
||||||
|
config := AppConfig{
|
||||||
|
SyncInterval: 60,
|
||||||
|
ShouldCache: true,
|
||||||
|
Scripts: []string{"all"},
|
||||||
|
}
|
||||||
|
file, err := os.Create(configPath + "/config.toml")
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
encoder := toml.NewEncoder(file)
|
||||||
|
err = encoder.Encode(config)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func printConfig() {
|
||||||
|
logger.Debug("Loaded config:")
|
||||||
|
logger.Debug("Sync time: ", appConfig.SyncInterval)
|
||||||
|
logger.Debug("Should cache: ", appConfig.ShouldCache)
|
||||||
|
logger.Debug("Scripts: ", appConfig.Scripts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadAppConfig() {
|
||||||
|
configPath := configdir.LocalConfig("rsslair")
|
||||||
|
err := configdir.MakePath(configPath)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(configPath + "/config.toml"); os.IsNotExist(err) {
|
||||||
|
logger.Info("Config file not found, creating default config")
|
||||||
|
createDefaultConfig(configPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(configPath + "/config.toml")
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
decoder := toml.NewDecoder(file)
|
||||||
|
err = decoder.Decode(&appConfig)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
}
|
||||||
|
printConfig()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,329 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "github.com/mattn/go-sqlite3"
|
||||||
|
lua "github.com/yuin/gopher-lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
const luaDatabaseTypeName = "sqlite3"
|
||||||
|
|
||||||
|
var db *sql.DB
|
||||||
|
|
||||||
|
var databaseMethods = map[string]lua.LGFunction{
|
||||||
|
"check": check,
|
||||||
|
"insert": insert_rss,
|
||||||
|
"getRss": GetRssFromDb,
|
||||||
|
}
|
||||||
|
|
||||||
|
func registerDatabaseType(L *lua.LState) {
|
||||||
|
logger.Debug("Registering database type")
|
||||||
|
mt := L.NewTypeMetatable("sqlite3")
|
||||||
|
L.SetGlobal("sqlite3", mt)
|
||||||
|
L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), databaseMethods))
|
||||||
|
|
||||||
|
// Create an instance of the database
|
||||||
|
ud := InitDb(L)
|
||||||
|
L.SetGlobal("db", ud)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the first lua argument is a *LUserData
|
||||||
|
func checkDatabase(L *lua.LState) *sql.DB {
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
if v, ok := ud.Value.(*sql.DB); ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
L.ArgError(1, "sqlite3 expected")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Constructor for new database userdata
|
||||||
|
func InitDb(L *lua.LState) *lua.LUserData {
|
||||||
|
ud := L.NewUserData()
|
||||||
|
|
||||||
|
// Create a database connection
|
||||||
|
database, err := sql.Open("sqlite3", "./rss.db")
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error opening database: ", err)
|
||||||
|
} // Create the table with the rss schema if this is the first time
|
||||||
|
// TODO: Check if the table exists first
|
||||||
|
sqlStmt := `
|
||||||
|
CREATE TABLE IF NOT EXISTS rss (id INTEGER PRIMARY KEY, title TEXT, link TEXT, description TEXT, pubDate TEXT, guid TEXT UNIQUE, category TEXT, read INTEGER DEFAULT 0);
|
||||||
|
`
|
||||||
|
_, err = database.Exec(sqlStmt)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error creating table: ", err)
|
||||||
|
}
|
||||||
|
ud.Value = database // Store the database connection in the userdata
|
||||||
|
db = database
|
||||||
|
|
||||||
|
_CreateRSSFeedDbTable(db)
|
||||||
|
|
||||||
|
L.SetMetatable(ud, L.GetTypeMetatable(luaDatabaseTypeName)) // Set the metatable for the userdata
|
||||||
|
return ud // Return the number of values pushed onto the stack
|
||||||
|
}
|
||||||
|
|
||||||
|
func _CreateRSSFeedDbTable(db *sql.DB) {
|
||||||
|
// Create the table with the rss schema if this is the first time
|
||||||
|
sqlStmt := `
|
||||||
|
CREATE TABLE IF NOT EXISTS feed (id INTEGER PRIMARY KEY, title TEXT, link TEXT UNIQUE, description TEXT, lastSyncTime TEXT);
|
||||||
|
`
|
||||||
|
_, err := db.Exec(sqlStmt)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error creating table: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// TODO: Check for duplicate entries before inserting
|
||||||
|
func insert_rss(L *lua.LState) int {
|
||||||
|
if L.GetTop() != 2 {
|
||||||
|
L.ArgError(2, "Expected RSS table")
|
||||||
|
}
|
||||||
|
item := L.CheckUserData(2)
|
||||||
|
if item == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
ud := item.Value.(*RssItem)
|
||||||
|
|
||||||
|
// Insert the rss into the database
|
||||||
|
sqlStmt := `
|
||||||
|
INSERT INTO rss (title, link, description, pubDate, guid, category)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)
|
||||||
|
`
|
||||||
|
_, err := db.Exec(sqlStmt, ud.Title, ud.Link, ud.Description, ud.PubDate, ud.Guid, ud.Category)
|
||||||
|
if err != nil && err.Error() != "UNIQUE constraint failed: rss.guid" {
|
||||||
|
logger.Error("Error inserting into table: ", err)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetRssFeed(link string) {
|
||||||
|
// Get the rss feed from the database
|
||||||
|
sqlStmt := `
|
||||||
|
SELECT title, link, description, lastSyncTime FROM feed WHERE link = ?
|
||||||
|
`
|
||||||
|
rows, err := db.Query(sqlStmt, link)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error querying table: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func check(L *lua.LState) int {
|
||||||
|
// Pass in a table of RSSItem
|
||||||
|
if L.GetTop() != 2 {
|
||||||
|
L.ArgError(2, "Expected RSS table")
|
||||||
|
}
|
||||||
|
items := L.CheckTable(2)
|
||||||
|
if items == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a map of the rss items
|
||||||
|
rssMap := make(map[string]*RssItem)
|
||||||
|
items.ForEach(func(_, value lua.LValue) {
|
||||||
|
item := value.(*lua.LUserData)
|
||||||
|
if v, ok := item.Value.(*RssItem); ok {
|
||||||
|
rssMap[v.Guid] = v
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// SELECT * FROM your_table WHERE your_column_name IN (value1, value2, ..., valueN);
|
||||||
|
// Create a string with all the guids
|
||||||
|
guids := make([]string, 0, len(rssMap))
|
||||||
|
for guid := range rssMap {
|
||||||
|
id := fmt.Sprintf("%q", guid)
|
||||||
|
guids = append(guids, id)
|
||||||
|
}
|
||||||
|
values := strings.Join(guids, ",")
|
||||||
|
sqlStmt := `
|
||||||
|
SELECT guid FROM rss WHERE guid IN (` + values + `)
|
||||||
|
`
|
||||||
|
|
||||||
|
// Return to lua the values that do not exist in the database
|
||||||
|
rows, err := db.Query(sqlStmt)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error querying table: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create array to store the previous guids
|
||||||
|
prevGuids := make([]string, 0, len(rssMap))
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var guid string
|
||||||
|
err = rows.Scan(&guid)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error scanning table: ", err)
|
||||||
|
}
|
||||||
|
prevGuids = append(prevGuids, guid)
|
||||||
|
delete(rssMap, guid)
|
||||||
|
}
|
||||||
|
|
||||||
|
table := L.NewTable()
|
||||||
|
prevTable := L.NewTable()
|
||||||
|
for _, v := range rssMap {
|
||||||
|
ud := L.NewUserData()
|
||||||
|
ud.Value = v
|
||||||
|
L.SetMetatable(ud, L.GetTypeMetatable(luaRSSItemTypeName))
|
||||||
|
table.Append(ud)
|
||||||
|
}
|
||||||
|
for _, v := range prevGuids {
|
||||||
|
prevTable.Append(lua.LString(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
L.Push(prevTable)
|
||||||
|
L.Push(table)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetRssFromDb(L *lua.LState) int {
|
||||||
|
// Pass in a table of RSSItem
|
||||||
|
if L.GetTop() != 2 {
|
||||||
|
L.ArgError(2, "Expected RSS table")
|
||||||
|
}
|
||||||
|
items := L.CheckTable(2)
|
||||||
|
if items == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create an array of the rss items
|
||||||
|
arr := make([]string, 0, items.Len())
|
||||||
|
items.ForEach(func(_, value lua.LValue) {
|
||||||
|
item := value.String()
|
||||||
|
id := fmt.Sprintf("%q", item)
|
||||||
|
arr = append(arr, id)
|
||||||
|
})
|
||||||
|
|
||||||
|
values := strings.Join(arr, ",")
|
||||||
|
sqlStmt := `
|
||||||
|
SELECT title, link, description, pubDate, guid, category FROM rss WHERE guid IN (` + values + `)
|
||||||
|
`
|
||||||
|
rows, err := db.Query(sqlStmt)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error querying table: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
table := L.NewTable()
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
item := RssItem{}
|
||||||
|
err = rows.Scan(&item.Title, &item.Link, &item.Description, &item.PubDate, &item.Guid, &item.Category)
|
||||||
|
// TODO: Handle this error better
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error scanning table: ", err)
|
||||||
|
}
|
||||||
|
ud := L.NewUserData()
|
||||||
|
ud.Value = &item
|
||||||
|
L.SetMetatable(ud, L.GetTypeMetatable(luaRSSItemTypeName))
|
||||||
|
table.Append(ud)
|
||||||
|
}
|
||||||
|
L.Push(table)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func get_rss_entry(L *lua.LState) int {
|
||||||
|
if L.GetTop() != 2 {
|
||||||
|
L.ArgError(2, "Expected RSS table")
|
||||||
|
}
|
||||||
|
// This function should only be getting data from the database
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func get_rss_feed(url string) RssFeed {
|
||||||
|
logger.Trace("===Executing get_rss_feed for url: ", url, "===")
|
||||||
|
// Get the rss feed from the database
|
||||||
|
sqlStmt := `
|
||||||
|
SELECT title, link, description, lastSyncTime FROM feed WHERE link = ?
|
||||||
|
`
|
||||||
|
rows, err := db.Query(sqlStmt, url)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error querying table: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rssFeed RssFeed
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
err = rows.Scan(&rssFeed.Title, &rssFeed.Link, &rssFeed.Content, &rssFeed.LastSyncTime)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error scanning table: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rssFeed
|
||||||
|
}
|
||||||
|
|
||||||
|
func insert_rss_feed(rssFeed RssFeed) {
|
||||||
|
sqlStmt := `
|
||||||
|
INSERT INTO feed (title, link, description, lastSyncTime)
|
||||||
|
VALUES (?, ?, ?, ?)
|
||||||
|
`
|
||||||
|
_, err := db.Exec(sqlStmt, rssFeed.Title, rssFeed.Link, rssFeed.Content, rssFeed.LastSyncTime)
|
||||||
|
if err != nil && err.Error() != "UNIQUE constraint failed: feed.link" {
|
||||||
|
logger.Error("Error inserting into table: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func update_rss_feed(rssFeed RssFeed) {
|
||||||
|
sqlStmt := `
|
||||||
|
UPDATE feed SET description = ?, lastSyncTime = ? WHERE link = ?
|
||||||
|
`
|
||||||
|
_, err := db.Exec(sqlStmt, rssFeed.Content, rssFeed.LastSyncTime, rssFeed.Link)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error updating table: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func DoesRssFeedExist(link string) bool {
|
||||||
|
sqlStmt := `
|
||||||
|
SELECT EXISTS(SELECT 1 FROM feed WHERE link = ? LIMIT 1)
|
||||||
|
`
|
||||||
|
rows, err := db.Query(sqlStmt, link)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error querying table: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var exists bool
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
err = rows.Scan(&exists)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error scanning table: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return exists
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetLastSyncTime(link string) (time.Time, error) {
|
||||||
|
sqlStmt := `
|
||||||
|
SELECT lastSyncTime FROM feed WHERE link = ?
|
||||||
|
`
|
||||||
|
rows, err := db.Query(sqlStmt, link)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error querying table: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastSyncTime string
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
err = rows.Scan(&lastSyncTime)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error scanning table: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return time.Parse(time.RFC3339, lastSyncTime)
|
||||||
|
}
|
||||||
|
|
||||||
|
func CloseDb() {
|
||||||
|
logger.Debug("Closing database")
|
||||||
|
err := db.Close()
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error closing database: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
version: "3.5"
|
||||||
|
services:
|
||||||
|
rsslair:
|
||||||
|
image: gitea.lan/christopher/rsslair
|
||||||
|
container_name: rsslair
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
volumes:
|
||||||
|
- ./rss.db:/app/rss.db
|
||||||
|
environment:
|
||||||
|
- LOG_LEVEL=debug
|
||||||
|
restart: unless-stopped
|
||||||
|
|
@ -0,0 +1,48 @@
|
||||||
|
module github.com/christopher876/rsslair
|
||||||
|
|
||||||
|
go 1.21.0
|
||||||
|
|
||||||
|
toolchain go1.21.5
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/PuerkitoBio/goquery v1.8.1
|
||||||
|
github.com/banzaicloud/logrus-runtime-formatter v0.0.0-20190729070250-5ae5475bae5e
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.19
|
||||||
|
github.com/sirupsen/logrus v1.9.3
|
||||||
|
github.com/yuin/gopher-lua v1.1.1
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/17xande/configdir v0.0.0-20230822134354-9441875917e7 // indirect
|
||||||
|
github.com/BurntSushi/toml v1.2.1 // indirect
|
||||||
|
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||||
|
github.com/antchfx/htmlquery v1.2.3 // indirect
|
||||||
|
github.com/antchfx/xmlquery v1.2.4 // indirect
|
||||||
|
github.com/antchfx/xpath v1.1.8 // indirect
|
||||||
|
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df // indirect
|
||||||
|
github.com/gobwas/glob v0.2.3 // indirect
|
||||||
|
github.com/gocolly/colly/v2 v2.1.0 // indirect
|
||||||
|
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||||
|
github.com/golang/protobuf v1.4.2 // indirect
|
||||||
|
github.com/google/go-cmp v0.5.9 // indirect
|
||||||
|
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||||
|
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
|
||||||
|
github.com/sergi/go-diff v1.1.0 // indirect
|
||||||
|
github.com/temoto/robotstxt v1.1.1 // indirect
|
||||||
|
golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 // indirect
|
||||||
|
golang.org/x/mod v0.14.0 // indirect
|
||||||
|
golang.org/x/net v0.16.0 // indirect
|
||||||
|
golang.org/x/sync v0.4.0 // indirect
|
||||||
|
golang.org/x/sys v0.15.0 // indirect
|
||||||
|
golang.org/x/telemetry v0.0.0-20231114163143-69313e640400 // indirect
|
||||||
|
golang.org/x/term v0.15.0 // indirect
|
||||||
|
golang.org/x/text v0.13.0 // indirect
|
||||||
|
golang.org/x/tools v0.14.1-0.20231114185516-c9d3e7de13fd // indirect
|
||||||
|
golang.org/x/vuln v1.0.1 // indirect
|
||||||
|
google.golang.org/appengine v1.6.6 // indirect
|
||||||
|
google.golang.org/protobuf v1.24.0 // indirect
|
||||||
|
honnef.co/go/tools v0.4.5 // indirect
|
||||||
|
mvdan.cc/gofumpt v0.4.0 // indirect
|
||||||
|
mvdan.cc/xurls/v2 v2.4.0 // indirect
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,210 @@
|
||||||
|
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
|
github.com/17xande/configdir v0.0.0-20230822134354-9441875917e7 h1:C5wjx8RnxNpSCQjMX4NWfeWjkmlaRWk1Sr/fQ0dR4oI=
|
||||||
|
github.com/17xande/configdir v0.0.0-20230822134354-9441875917e7/go.mod h1:QdroZvxv+xvY8TtnEFTdH+IxNyiEO59JJpfKjTX74+E=
|
||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
|
||||||
|
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||||
|
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
|
||||||
|
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||||
|
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
|
||||||
|
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||||
|
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||||
|
github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
|
||||||
|
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
|
||||||
|
github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4=
|
||||||
|
github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
|
||||||
|
github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||||
|
github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk=
|
||||||
|
github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||||
|
github.com/banzaicloud/logrus-runtime-formatter v0.0.0-20190729070250-5ae5475bae5e h1:ZOnKnYG1LLgq4W7wZUYj9ntn3RxQ65EZyYqdtFpP2Dw=
|
||||||
|
github.com/banzaicloud/logrus-runtime-formatter v0.0.0-20190729070250-5ae5475bae5e/go.mod h1:hEvEpPmuwKO+0TbrDQKIkmX0gW2s2waZHF8pIhEEmpM=
|
||||||
|
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
|
||||||
|
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
|
||||||
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
|
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||||
|
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||||
|
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||||
|
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||||
|
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
|
||||||
|
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
||||||
|
github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=
|
||||||
|
github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=
|
||||||
|
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||||
|
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||||
|
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
|
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||||
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||||
|
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||||
|
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||||
|
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||||
|
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||||
|
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||||
|
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||||
|
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||||
|
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
|
||||||
|
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||||
|
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||||
|
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||||
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||||
|
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
|
||||||
|
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||||
|
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||||
|
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
|
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
|
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||||
|
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
|
||||||
|
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||||
|
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||||
|
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||||
|
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||||
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
|
||||||
|
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||||
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
|
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||||
|
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
|
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
|
golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 h1:2O2DON6y3XMJiQRAS1UWU+54aec2uopH3x7MAiqGW6Y=
|
||||||
|
golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
|
||||||
|
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
|
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||||
|
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
|
||||||
|
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||||
|
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||||
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
|
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
|
golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
|
||||||
|
golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
|
||||||
|
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||||
|
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
|
||||||
|
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
|
||||||
|
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/telemetry v0.0.0-20231114163143-69313e640400 h1:brbkEFfGwNGAEkykUOcryE/JiHUMMJouzE0fWWmz/QU=
|
||||||
|
golang.org/x/telemetry v0.0.0-20231114163143-69313e640400/go.mod h1:P6hMdmAcoG7FyATwqSr6R/U0n7yeXNP/QXeRlxb1szE=
|
||||||
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
|
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||||
|
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
|
||||||
|
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
|
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
|
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||||
|
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||||
|
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
|
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||||
|
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||||
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
|
golang.org/x/tools v0.14.1-0.20231114185516-c9d3e7de13fd h1:Oku7E+OCrXHyst1dG1z10etCTxewCHXNFLRlyMPbh3w=
|
||||||
|
golang.org/x/tools v0.14.1-0.20231114185516-c9d3e7de13fd/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||||
|
golang.org/x/tools/gopls v0.14.2 h1:sIw6vjZiuQ9S7s0auUUkHlWgsCkKZFWDHmrge8LYsnc=
|
||||||
|
golang.org/x/tools/gopls v0.14.2/go.mod h1:o2s+suLlFye+MHxEofsxJPkBQ/kT/ucP4iJV3ohjGEQ=
|
||||||
|
golang.org/x/vuln v1.0.1 h1:KUas02EjQK5LTuIx1OylBQdKKZ9jeugs+HiqO5HormU=
|
||||||
|
golang.org/x/vuln v1.0.1/go.mod h1:bb2hMwln/tqxg32BNY4CcxHWtHXuYa3SbIBmtsyjxtM=
|
||||||
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||||
|
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
|
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
|
||||||
|
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
|
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||||
|
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||||
|
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||||
|
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||||
|
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||||
|
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||||
|
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||||
|
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||||
|
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
|
||||||
|
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.4.5 h1:YGD4H+SuIOOqsyoLOpZDWcieM28W47/zRO7f+9V3nvo=
|
||||||
|
honnef.co/go/tools v0.4.5/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k=
|
||||||
|
mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM=
|
||||||
|
mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ=
|
||||||
|
mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc=
|
||||||
|
mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg=
|
||||||
|
|
@ -0,0 +1,171 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
lua "github.com/yuin/gopher-lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
type HtmlParser struct {
|
||||||
|
Doc *goquery.Document
|
||||||
|
}
|
||||||
|
|
||||||
|
const luaHtmlParserTypeName = "html"
|
||||||
|
var luaHtmlParserMethods = map[string]lua.LGFunction{
|
||||||
|
"select": select_html_node,
|
||||||
|
"remove": remove_html_node,
|
||||||
|
"get": get_document,
|
||||||
|
"get_attribute": get_html_node_attribute,
|
||||||
|
"rewrite_nodes": rewrite_html_nodes,
|
||||||
|
}
|
||||||
|
|
||||||
|
func registerHtmlParserType(L *lua.LState) {
|
||||||
|
logger.Debug("Registering html type")
|
||||||
|
mt := L.NewTypeMetatable(luaHtmlParserTypeName)
|
||||||
|
L.SetGlobal(luaHtmlParserTypeName, mt)
|
||||||
|
L.SetField(mt, "new", L.NewFunction(newHtmlParser))
|
||||||
|
L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), luaHtmlParserMethods))
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkHtmlParser(L *lua.LState) *HtmlParser {
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
if v, ok := ud.Value.(*HtmlParser); ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
L.ArgError(1, "html_parser expected")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newHtmlParser(L *lua.LState) int {
|
||||||
|
source := L.CheckString(1)
|
||||||
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(source))
|
||||||
|
if err != nil {
|
||||||
|
logger.Error(err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return doc as userdata
|
||||||
|
ud := L.NewUserData()
|
||||||
|
ud.Value = doc
|
||||||
|
L.SetMetatable(ud, L.GetTypeMetatable(luaHtmlParserTypeName))
|
||||||
|
L.Push(ud)
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
func select_html_node(L *lua.LState) int {
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
selector := L.CheckString(2)
|
||||||
|
doc, ok := ud.Value.(*goquery.Document)
|
||||||
|
if !ok {
|
||||||
|
logger.Error("Expected html_parser userdata")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
var result []string
|
||||||
|
doc.Find(selector).Each(func(_ int, s *goquery.Selection) {
|
||||||
|
pHtml, err := s.Html()
|
||||||
|
if err != nil {
|
||||||
|
logger.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
result = append(result, pHtml)
|
||||||
|
})
|
||||||
|
|
||||||
|
L.Push(lua.LString(strings.Join(result, "\n")))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func remove_html_node(L *lua.LState) int {
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
selector := L.CheckString(2)
|
||||||
|
doc, ok := ud.Value.(*goquery.Document)
|
||||||
|
if !ok {
|
||||||
|
logger.Error("Expected html_parser userdata")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
doc.Find(selector).Each(func(_ int, s *goquery.Selection) {
|
||||||
|
s.Remove()
|
||||||
|
})
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func get_html_node_attribute(L *lua.LState) int {
|
||||||
|
// Get the node, for example <img src="local/image.jpg"> -> <img src="http://example.com/image.jpg">
|
||||||
|
// Get the attribute, src
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
node := L.CheckString(2)
|
||||||
|
attribute := L.CheckString(3)
|
||||||
|
|
||||||
|
doc, ok := ud.Value.(*goquery.Document)
|
||||||
|
if !ok {
|
||||||
|
logger.Error("Expected html_parser userdata")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
result := []string{}
|
||||||
|
doc.Find(node).Each(func(_ int, s *goquery.Selection) {
|
||||||
|
pHtml, ok := s.Attr(attribute)
|
||||||
|
if !ok {
|
||||||
|
logger.Error("Could not find attribute: ", attribute, " in node: ", node)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
result = append(result, pHtml)
|
||||||
|
})
|
||||||
|
|
||||||
|
table := L.NewTable()
|
||||||
|
for i := range result {
|
||||||
|
table.Append(lua.LString(result[i]))
|
||||||
|
}
|
||||||
|
L.Push(table)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// article:rewrite("img", "src", {"http://example.com/image.jpg", "http://example.com/image2.jpg"})
|
||||||
|
func rewrite_html_nodes(L *lua.LState) int {
|
||||||
|
// Get the node, for example <img src="local/image.jpg"> -> <img src="http://example.com/image.jpg">
|
||||||
|
// Get the attribute, src
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
node := L.CheckString(2)
|
||||||
|
attribute := L.CheckString(3)
|
||||||
|
rewriteArr := L.CheckTable(4)
|
||||||
|
|
||||||
|
doc, ok := ud.Value.(*goquery.Document)
|
||||||
|
if !ok {
|
||||||
|
logger.Error("Expected html_parser userdata")
|
||||||
|
}
|
||||||
|
|
||||||
|
rewriteStrings := []string{}
|
||||||
|
for i := 1; i <= rewriteArr.Len(); i++ {
|
||||||
|
rewriteStrings = append(rewriteStrings, rewriteArr.RawGetInt(i).String())
|
||||||
|
}
|
||||||
|
i := 0
|
||||||
|
doc.Find(node).Each(func(_ int, s *goquery.Selection) {
|
||||||
|
rewrite := rewriteStrings[i]
|
||||||
|
s.SetAttr(attribute, rewrite)
|
||||||
|
logger.Debug("Rewrote: ", attribute, " to: ", rewrite)
|
||||||
|
i++
|
||||||
|
})
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func get_document(L *lua.LState) int {
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
doc, ok := ud.Value.(*goquery.Document)
|
||||||
|
if !ok {
|
||||||
|
logger.Error("Expected html_parser userdata")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
html, err := doc.Html()
|
||||||
|
if err != nil {
|
||||||
|
logger.Error(err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
L.Push(lua.LString(html))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,257 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
lua "github.com/yuin/gopher-lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
func InitLua() *lua.LState {
|
||||||
|
logger.Info("Initializing lua")
|
||||||
|
L := lua.NewState()
|
||||||
|
// load_log_library(L)
|
||||||
|
bind_lua_functions(L)
|
||||||
|
LoadScripts(L)
|
||||||
|
return L
|
||||||
|
}
|
||||||
|
|
||||||
|
func LoadLogFunctions(L *lua.LState) {
|
||||||
|
table := L.NewTable()
|
||||||
|
L.SetFuncs(table, map[string]lua.LGFunction{
|
||||||
|
"info": luaLogInfo,
|
||||||
|
"debug": luaLogDebug,
|
||||||
|
"error": luaLogError,
|
||||||
|
"fatal": luaLogFatal,
|
||||||
|
})
|
||||||
|
L.SetGlobal("log", table)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadScript(L *lua.LState, script string) error {
|
||||||
|
if err := L.DoFile(script); err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
logger.Debug("Loaded script: ", script)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func LoadScripts(L *lua.LState) error {
|
||||||
|
// iterate over the scripts directory and load each script
|
||||||
|
files, err := os.ReadDir("scripts")
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
loadAllScripts := appConfig.Scripts[0] == "all" || appConfig.Scripts[0] == ""
|
||||||
|
if loadAllScripts {
|
||||||
|
for _, file := range files {
|
||||||
|
var extension = filepath.Ext(file.Name())
|
||||||
|
if file.IsDir() || extension != ".lua" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := loadScript(L, "scripts/"+file.Name()); err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for _, script := range appConfig.Scripts {
|
||||||
|
if err := loadScript(L, "scripts/"+script+".lua"); err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func execute_route(L *lua.LState, route string, script string) (string, error) {
|
||||||
|
logger.Debug("Executing route: ", route)
|
||||||
|
|
||||||
|
// Get table by script name
|
||||||
|
table := L.GetGlobal(script)
|
||||||
|
// Get the function from the table
|
||||||
|
fn := L.GetField(table, "route")
|
||||||
|
// TODO: The message should say something else
|
||||||
|
if fn.Type() != lua.LTFunction {
|
||||||
|
logger.Error("route function was not found in script: ", script)
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := L.CallByParam(lua.P{
|
||||||
|
Fn: fn,
|
||||||
|
NRet: 1,
|
||||||
|
Protect: true,
|
||||||
|
}); err != nil {
|
||||||
|
logger.Error(err)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
ret := L.Get(-1)
|
||||||
|
responses[route] = strings.Clone(ret.String())
|
||||||
|
|
||||||
|
return ret.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func add_html_parser(L *lua.LState) int {
|
||||||
|
// Create a table for the html_parser add_functions
|
||||||
|
html_table := L.NewTable()
|
||||||
|
L.SetFuncs(html_table, map[string]lua.LGFunction{
|
||||||
|
"parse": newHtmlParser,
|
||||||
|
"select": select_html_node,
|
||||||
|
})
|
||||||
|
L.Push(html_table)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func bind_lua_functions(L *lua.LState) error {
|
||||||
|
// Add global functions to lua
|
||||||
|
LoadLogFunctions(L)
|
||||||
|
|
||||||
|
L.SetGlobal("add_route", L.NewFunction(lua_add_route))
|
||||||
|
L.SetGlobal("get", L.NewFunction(get_request))
|
||||||
|
L.SetGlobal("parse_xml_feed", L.NewFunction(parse_xml_feed))
|
||||||
|
L.SetGlobal("create_rss_feed", L.NewFunction(create_rss_feed))
|
||||||
|
|
||||||
|
// Register Golang structs with lua
|
||||||
|
registerDatabaseType(L)
|
||||||
|
registerRSSItemType(L)
|
||||||
|
registerHtmlParserType(L)
|
||||||
|
registerRssImageType(L)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func lua_add_route(L *lua.LState) int {
|
||||||
|
// TODO: Check that the parameters are correct
|
||||||
|
// Handle errors
|
||||||
|
script := L.ToString(1)
|
||||||
|
route := L.ToString(2)
|
||||||
|
table := L.NewTable()
|
||||||
|
|
||||||
|
L.SetGlobal(script, table)
|
||||||
|
|
||||||
|
routes[route] = script
|
||||||
|
responses[route] = ""
|
||||||
|
logger.Debug("Adding route: ", route, " -> ", script)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func get_request(L *lua.LState) int {
|
||||||
|
url := L.ToString(1)
|
||||||
|
logger.Debug("GET: ", url)
|
||||||
|
resp, err := http.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error(err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(string(body)))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func parse_xml_feed(L *lua.LState) int {
|
||||||
|
xml_str := L.ToString(1)
|
||||||
|
|
||||||
|
var rss RssRoot
|
||||||
|
err := xml.Unmarshal([]byte(xml_str), &rss)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
feed_items := L.NewTable()
|
||||||
|
items := rss.Channel.Items
|
||||||
|
|
||||||
|
for i := range items {
|
||||||
|
ud := L.NewUserData()
|
||||||
|
ud.Value = &items[i]
|
||||||
|
L.SetMetatable(ud, L.GetTypeMetatable(luaRSSItemTypeName))
|
||||||
|
feed_items.Append(ud)
|
||||||
|
}
|
||||||
|
L.Push(feed_items)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Move this into the rss table
|
||||||
|
func create_rss_feed(L *lua.LState) int {
|
||||||
|
//TODO: Check that the parameters are correct
|
||||||
|
feed := RssRoot{}
|
||||||
|
|
||||||
|
//TODO: Bind RssImage to lua and pass it in
|
||||||
|
// Also needs to be added to the database and then re-served through the server
|
||||||
|
title := L.CheckString(1)
|
||||||
|
link := L.CheckString(2)
|
||||||
|
image := L.CheckUserData(3)
|
||||||
|
// TODO: Channel needs to also be passed in
|
||||||
|
feed.Channel = RssChannel{
|
||||||
|
Title: title,
|
||||||
|
Image: *image.Value.(*RssImage),
|
||||||
|
Link: link,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Append the items to the feed
|
||||||
|
// Notably the <channel> tag as <item>s
|
||||||
|
entries := L.CheckTable(4)
|
||||||
|
if entries.Len() == 0 {
|
||||||
|
logger.Error("No entries to create rss feed found in table")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 1; i <= entries.Len(); i++ {
|
||||||
|
entry := entries.RawGetInt(i)
|
||||||
|
ud := entry.(*lua.LUserData)
|
||||||
|
item := ud.Value.(*RssItem)
|
||||||
|
feed.Channel.Items = append(feed.Channel.Items, *item)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sort by date
|
||||||
|
sort.Sort(ByPubDate(feed.Channel.Items))
|
||||||
|
|
||||||
|
output, err := xml.MarshalIndent(feed, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
L.Push(lua.LString(string(output)))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func luaLogInfo(L *lua.LState) int {
|
||||||
|
loc := L.Where(1)
|
||||||
|
msg := L.ToString(1)
|
||||||
|
logger.Info(loc, msg)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func luaLogDebug(L *lua.LState) int {
|
||||||
|
// Get the where and message
|
||||||
|
loc := L.Where(1)
|
||||||
|
msg := L.ToString(1)
|
||||||
|
logger.Debug(loc, msg)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func luaLogError(L *lua.LState) int {
|
||||||
|
loc := L.Where(1)
|
||||||
|
msg := L.ToString(1)
|
||||||
|
logger.Error(loc, msg)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func luaLogFatal(L *lua.LState) int {
|
||||||
|
loc := L.Where(1)
|
||||||
|
msg := L.ToString(1)
|
||||||
|
logger.Fatal(loc, msg)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,141 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
runtime "github.com/banzaicloud/logrus-runtime-formatter"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
var logger = logrus.New()
|
||||||
|
var appConfig AppConfig
|
||||||
|
|
||||||
|
// TODO: This function needs to be refactored
|
||||||
|
func setLogLevel() {
|
||||||
|
level := os.Getenv("LOG_LEVEL")
|
||||||
|
verbosity := os.Getenv("VERBOSE")
|
||||||
|
|
||||||
|
if verbosity == "1" {
|
||||||
|
formatter := runtime.Formatter{ChildFormatter: &logrus.TextFormatter{
|
||||||
|
FullTimestamp: true,
|
||||||
|
}}
|
||||||
|
formatter.Line = true
|
||||||
|
|
||||||
|
logger.SetFormatter(&formatter)
|
||||||
|
}
|
||||||
|
|
||||||
|
//TODO: This should be in its own function
|
||||||
|
logFile, err := os.OpenFile("rsslair.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
}
|
||||||
|
mw := io.MultiWriter(os.Stdout, logFile)
|
||||||
|
logger.SetOutput(mw)
|
||||||
|
|
||||||
|
switch strings.ToLower(level) {
|
||||||
|
case "debug":
|
||||||
|
logger.SetLevel(logrus.DebugLevel)
|
||||||
|
case "info":
|
||||||
|
logger.SetLevel(logrus.InfoLevel)
|
||||||
|
case "warn":
|
||||||
|
logger.SetLevel(logrus.WarnLevel)
|
||||||
|
case "error":
|
||||||
|
logger.SetLevel(logrus.ErrorLevel)
|
||||||
|
case "trace":
|
||||||
|
logger.SetLevel(logrus.TraceLevel)
|
||||||
|
default:
|
||||||
|
logger.SetLevel(logrus.InfoLevel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
setLogLevel()
|
||||||
|
loadAppConfig()
|
||||||
|
|
||||||
|
lua := InitLua()
|
||||||
|
defer CloseDb()
|
||||||
|
defer lua.Close()
|
||||||
|
|
||||||
|
// Create a channel to listen for signals
|
||||||
|
// Listen for SIGINT and SIGTERM
|
||||||
|
sig_channel := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sig_channel, os.Interrupt, syscall.SIGTERM)
|
||||||
|
|
||||||
|
address := "0.0.0.0"
|
||||||
|
port := "8080"
|
||||||
|
server := start_server(address, port, lua)
|
||||||
|
|
||||||
|
// TODO: Make this look nice
|
||||||
|
// Create channel to listen for input
|
||||||
|
go func() {
|
||||||
|
// disable input buffering
|
||||||
|
exec.Command("stty", "-F", "/dev/tty", "cbreak", "min", "1").Run()
|
||||||
|
// do not display entered characters on the screen
|
||||||
|
exec.Command("stty", "-F", "/dev/tty", "-echo").Run()
|
||||||
|
// restore the echoing state when exiting
|
||||||
|
defer exec.Command("stty", "-F", "/dev/tty", "echo").Run()
|
||||||
|
|
||||||
|
var b = make([]byte, 1)
|
||||||
|
for {
|
||||||
|
os.Stdin.Read(b)
|
||||||
|
switch char := string(b); char {
|
||||||
|
case "r":
|
||||||
|
fmt.Println("Reloading scripts")
|
||||||
|
routes = make(map[string]string)
|
||||||
|
LoadScripts(lua)
|
||||||
|
case "s":
|
||||||
|
fmt.Println("Executing lua scripts")
|
||||||
|
execute_all_routes(lua)
|
||||||
|
case "d":
|
||||||
|
fmt.Println("Dumping response data")
|
||||||
|
for k, v := range responses {
|
||||||
|
leng := len(v)
|
||||||
|
if leng > 30 {
|
||||||
|
leng = 30
|
||||||
|
} else if leng == 0 {
|
||||||
|
fmt.Println(k, "No response data")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := v[:leng]
|
||||||
|
fmt.Println(k, msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
execute_all_routes(lua)
|
||||||
|
|
||||||
|
// TODO: Make this look nice in an actual function
|
||||||
|
hasRun := false
|
||||||
|
luaExecutionTimer := time.NewTicker(30 * time.Second)
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-luaExecutionTimer.C:
|
||||||
|
now := time.Now()
|
||||||
|
if now.Minute() == 55 && !hasRun {
|
||||||
|
hasRun = true
|
||||||
|
execute_all_routes(lua)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
hasRun = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
<-sig_channel // Block until SIGINT or SIGTERM is received
|
||||||
|
logger.Info("Shutting down now")
|
||||||
|
if err := server.Shutdown(context.TODO()); err != nil {
|
||||||
|
logger.Fatal("Error shutting down server: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
lua "github.com/yuin/gopher-lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
var routes = make(map[string]string)
|
||||||
|
var responses = make(map[string]string)
|
||||||
|
|
||||||
|
func execute_all_routes(L *lua.LState) {
|
||||||
|
logger.Debug("Executing all scripts")
|
||||||
|
for path, script := range routes {
|
||||||
|
logger.Trace("Executing route: ", path)
|
||||||
|
execute_route(L, path, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func route(L *lua.LState, request *http.Request) (string, int) {
|
||||||
|
path := request.URL.Path
|
||||||
|
|
||||||
|
// Check if the path is in the routes map
|
||||||
|
if _, ok := routes[path]; !ok {
|
||||||
|
return "", http.StatusNotFound
|
||||||
|
}
|
||||||
|
// response, _ := execute_route(L, routes[path])
|
||||||
|
|
||||||
|
// TODO: This needs to actually get the right base path
|
||||||
|
response := responses[path]
|
||||||
|
|
||||||
|
return response, 200
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,382 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
lua "github.com/yuin/gopher-lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RssFeed struct {
|
||||||
|
Title string // This is the title of the rss feed
|
||||||
|
Link string // This is the link to original rss feed (if available)
|
||||||
|
Content string // This contains the entire rss xml
|
||||||
|
LastSyncTime string // This is the last time the rss feed was synced
|
||||||
|
}
|
||||||
|
|
||||||
|
type RssRoot struct {
|
||||||
|
XMLName xml.Name `xml:"rss"`
|
||||||
|
Channel RssChannel `xml:"channel"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type RssChannel struct {
|
||||||
|
Title string `xml:"title"`
|
||||||
|
Link string `xml:"link"`
|
||||||
|
Description string `xml:"description"`
|
||||||
|
Image RssImage `xml:"image"`
|
||||||
|
Items []RssItem `xml:"item"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type RssItem struct {
|
||||||
|
Title string `xml:"title"`
|
||||||
|
Link string `xml:"link"`
|
||||||
|
Description string `xml:"description"`
|
||||||
|
Category string `xml:"category"`
|
||||||
|
Guid string `xml:"guid"`
|
||||||
|
PubDate string `xml:"pubDate"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type RssImage struct {
|
||||||
|
Url string `xml:"url"`
|
||||||
|
Title string `xml:"title"`
|
||||||
|
Link string `xml:"link"`
|
||||||
|
}
|
||||||
|
|
||||||
|
const luaRSSGlobalTable = "rss"
|
||||||
|
const luaRSSItemTypeName = "RssItem"
|
||||||
|
const luaRSSImageTypeName = "RssImage"
|
||||||
|
|
||||||
|
var luaRSSItemMethods = map[string]lua.LGFunction{
|
||||||
|
"title": getSetRssItemTitle,
|
||||||
|
"link": getSetRssItemLink,
|
||||||
|
"description": getSetRssItemDescription,
|
||||||
|
"category": getSetRssItemCategory,
|
||||||
|
"guid": getSetRssItemGuid,
|
||||||
|
"pubDate": getSetRssItemPubDate,
|
||||||
|
}
|
||||||
|
|
||||||
|
var luaRSSImageMethods = map[string]lua.LGFunction{
|
||||||
|
"title": getSetRssImageTitle,
|
||||||
|
"url": getSetRssImageUrl,
|
||||||
|
"link": getSetRssImageLink,
|
||||||
|
}
|
||||||
|
|
||||||
|
func registerRSSItemType(L *lua.LState) {
|
||||||
|
logger.Debug("Registering RssItem type")
|
||||||
|
mt := L.NewTypeMetatable(luaRSSItemTypeName)
|
||||||
|
L.SetGlobal(luaRSSItemTypeName, mt)
|
||||||
|
L.SetField(mt, "new", L.NewFunction(newRSSItem))
|
||||||
|
L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), luaRSSItemMethods))
|
||||||
|
|
||||||
|
// Create a global table called "rss"
|
||||||
|
// This table will contain all of the rss helper functions
|
||||||
|
rssTable := L.NewTable()
|
||||||
|
L.SetGlobal(luaRSSGlobalTable, rssTable)
|
||||||
|
L.SetField(rssTable, "get", L.NewFunction(getRssFeedHttp))
|
||||||
|
L.SetField(rssTable, "merge", L.NewFunction(mergeRssItems))
|
||||||
|
L.SetField(rssTable, "limit", L.NewFunction(limitRssItems))
|
||||||
|
L.SetField(rssTable, "generate", L.NewFunction(generateRssFeed))
|
||||||
|
}
|
||||||
|
|
||||||
|
func registerRssImageType(L *lua.LState) {
|
||||||
|
logger.Debug("Registering RssImage type")
|
||||||
|
mt := L.NewTypeMetatable(luaRSSImageTypeName)
|
||||||
|
L.SetGlobal(luaRSSImageTypeName, mt)
|
||||||
|
L.SetField(mt, "new", L.NewFunction(newRSSImage))
|
||||||
|
L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), luaRSSImageMethods))
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRSSImage(L *lua.LState) int {
|
||||||
|
ud := L.NewUserData()
|
||||||
|
if L.GetTop() != 3 {
|
||||||
|
L.ArgError(3, "Expected 3 arguments")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// TODO: Handle the error better
|
||||||
|
title := L.CheckString(1)
|
||||||
|
url := L.CheckString(2)
|
||||||
|
link := L.CheckString(3)
|
||||||
|
item := RssImage{
|
||||||
|
Url: url,
|
||||||
|
Title: title,
|
||||||
|
Link: link,
|
||||||
|
}
|
||||||
|
ud.Value = &item
|
||||||
|
L.SetMetatable(ud, L.GetTypeMetatable(luaRSSImageTypeName))
|
||||||
|
L.Push(ud)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkRSSImage(L *lua.LState) *RssImage {
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
if v, ok := ud.Value.(*RssImage); ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
L.ArgError(1, "RssImage expected")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSetRssImageTitle(L *lua.LState) int {
|
||||||
|
i := checkRSSImage(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.Title = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.Title))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSetRssImageUrl(L *lua.LState) int {
|
||||||
|
i := checkRSSImage(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.Url = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.Url))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSetRssImageLink(L *lua.LState) int {
|
||||||
|
i := checkRSSImage(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.Link = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.Link))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkRSSItem(L *lua.LState) *RssItem {
|
||||||
|
ud := L.CheckUserData(1)
|
||||||
|
if v, ok := ud.Value.(*RssItem); ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
L.ArgError(1, "RssItem expected")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRSSItem(L *lua.LState) int {
|
||||||
|
ud := L.NewUserData()
|
||||||
|
|
||||||
|
if L.GetTop() != 1 {
|
||||||
|
L.ArgError(1, "Expected 1 argument")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
item := L.CheckUserData(1)
|
||||||
|
ud.Value = item
|
||||||
|
L.SetMetatable(ud, L.GetTypeMetatable(luaRSSItemTypeName))
|
||||||
|
L.Push(ud)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Getters and setters for the RssItem type
|
||||||
|
func getSetRssItemTitle(L *lua.LState) int {
|
||||||
|
i := checkRSSItem(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.Title = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.Title))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSetRssItemLink(L *lua.LState) int {
|
||||||
|
i := checkRSSItem(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.Link = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.Link))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSetRssItemDescription(L *lua.LState) int {
|
||||||
|
i := checkRSSItem(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.Description = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.Description))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSetRssItemCategory(L *lua.LState) int {
|
||||||
|
i := checkRSSItem(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.Category = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.Category))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSetRssItemGuid(L *lua.LState) int {
|
||||||
|
i := checkRSSItem(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.Guid = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.Guid))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSetRssItemPubDate(L *lua.LState) int {
|
||||||
|
i := checkRSSItem(L)
|
||||||
|
if L.GetTop() == 2 {
|
||||||
|
i.PubDate = L.CheckString(2)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(i.PubDate))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseRssFeed(body string) *RssRoot {
|
||||||
|
var rss RssRoot
|
||||||
|
err := xml.Unmarshal([]byte(body), &rss)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &rss
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetHTTPRequest(url string) string {
|
||||||
|
logger.Debug("GET: ", url)
|
||||||
|
resp, err := http.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error(err)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getRssFeedHttp(L *lua.LState) int {
|
||||||
|
if L.GetTop() != 2 {
|
||||||
|
L.ArgError(1, "Expected 1 argument")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// Get the url from the first argument
|
||||||
|
url := L.CheckString(2)
|
||||||
|
|
||||||
|
// Check if the rss feed exists in the database
|
||||||
|
var ret string
|
||||||
|
currentTime := time.Now().UTC()
|
||||||
|
if !appConfig.ShouldCache {
|
||||||
|
// If the cache option is disabled, then always fetch the latest entries from the website
|
||||||
|
logger.Debug("Cache option is disabled. Syncing: ", url)
|
||||||
|
body := GetHTTPRequest(url)
|
||||||
|
L.Push(lua.LString(body))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: There might be some cleanup opportunities here
|
||||||
|
if !DoesRssFeedExist(url) {
|
||||||
|
// If it does not exist in the database, then get the latest entries from the website and insert them into the database
|
||||||
|
logger.Debug("Rss feed does not exist in database. Syncing for the first time: ", url)
|
||||||
|
|
||||||
|
body := GetHTTPRequest(url)
|
||||||
|
feed := ParseRssFeed(body)
|
||||||
|
rssFeed := RssFeed{
|
||||||
|
Title: feed.Channel.Title,
|
||||||
|
Link: url,
|
||||||
|
Content: string(body),
|
||||||
|
LastSyncTime: currentTime.Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
insert_rss_feed(rssFeed)
|
||||||
|
ret = body
|
||||||
|
} else {
|
||||||
|
logger.Debug("Rss feed exists in database but sync is required: ", url)
|
||||||
|
body := GetHTTPRequest(url)
|
||||||
|
feed := ParseRssFeed(body)
|
||||||
|
rssFeed := RssFeed{
|
||||||
|
Title: feed.Channel.Title,
|
||||||
|
Link: url,
|
||||||
|
Content: string(body),
|
||||||
|
LastSyncTime: currentTime.Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
update_rss_feed(rssFeed)
|
||||||
|
ret = body
|
||||||
|
|
||||||
|
}
|
||||||
|
//TODO: If the cache option is disabled, then always fetch the latest entries from the website
|
||||||
|
L.Push(lua.LString(ret))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeRssItems(L *lua.LState) int {
|
||||||
|
table1 := L.CheckTable(1)
|
||||||
|
table2 := L.CheckTable(2)
|
||||||
|
|
||||||
|
if table1.Len() == 0 {
|
||||||
|
L.Push(table2)
|
||||||
|
return 1
|
||||||
|
} else if table2.Len() == 0 {
|
||||||
|
L.Push(table1)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
table2.ForEach(func(_ lua.LValue, value lua.LValue) {
|
||||||
|
table1.Append(value)
|
||||||
|
})
|
||||||
|
L.Push(table1)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func limitRssItems(L *lua.LState) int {
|
||||||
|
table := L.CheckTable(1)
|
||||||
|
limit := L.CheckInt(2)
|
||||||
|
|
||||||
|
for i := table.Len(); i > limit; i-- {
|
||||||
|
table.Remove(i)
|
||||||
|
}
|
||||||
|
L.Push(table)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateRssFeed(L *lua.LState) int {
|
||||||
|
// Creating a new rss feed from scratch
|
||||||
|
title := L.CheckString(1)
|
||||||
|
entries := L.CheckTable(2) // For right now this will only be a string array of the description
|
||||||
|
|
||||||
|
feed := RssRoot{}
|
||||||
|
feed.Channel = RssChannel{
|
||||||
|
Title: title,
|
||||||
|
Link: "http://example.com",
|
||||||
|
Description: "This is an example rss feed",
|
||||||
|
Image: RssImage{
|
||||||
|
Url: "http://example.com/image.jpg",
|
||||||
|
Title: "Example Image",
|
||||||
|
Link: "http://example.com",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Append the items to the feed
|
||||||
|
// Notably the <channel> tag as <item>s
|
||||||
|
items := []RssItem{}
|
||||||
|
for i := 1; i <= entries.Len(); i++ {
|
||||||
|
entry := entries.RawGetInt(i)
|
||||||
|
item := RssItem{
|
||||||
|
Title: "Novemeber 2023",
|
||||||
|
Link: "http://example.com",
|
||||||
|
Description: entry.String(),
|
||||||
|
Category: "Reverse Engineering",
|
||||||
|
Guid: "http://example.com",
|
||||||
|
PubDate: time.Now().UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
feed.Channel.Items = items
|
||||||
|
output, err := xml.MarshalIndent(feed, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatal(err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
L.Push(lua.LString(string(output)))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
type ByPubDate []RssItem
|
||||||
|
|
||||||
|
func (a ByPubDate) Len() int { return len(a) }
|
||||||
|
func (a ByPubDate) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||||
|
func (a ByPubDate) Less(i, j int) bool { return parsePubDate(a[i].PubDate).After(parsePubDate(a[j].PubDate)) }
|
||||||
|
|
||||||
|
func parsePubDate(pubDateStr string) time.Time {
|
||||||
|
// Add the appropriate time parsing logic based on your date format
|
||||||
|
// For example, if your date format is "Mon, 02 Jan 2006 15:04:05 -0700",
|
||||||
|
// you can use the following parsing code
|
||||||
|
parsedTime, _ := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", pubDateStr)
|
||||||
|
return parsedTime
|
||||||
|
}
|
||||||
26
run.bash
26
run.bash
|
|
@ -1,26 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
# Set the PORT, NO_SCHEDULER and CRON_SCHEDULE environment variables if not set
|
|
||||||
|
|
||||||
# RUST_LOG: The log level for the server
|
|
||||||
# PORT: The port to run the server on
|
|
||||||
# NO_SCHEDULER: If set to true, the scheduler will not run
|
|
||||||
# CRON_SCHEDULE: The cron schedule to run the scheduler on
|
|
||||||
|
|
||||||
RUST_LOG=${RUST_LOG:-'info'}
|
|
||||||
PORT=${PORT:-8080}
|
|
||||||
REDIS_HOST=${REDIS_HOST:-'redis'}
|
|
||||||
REDIS_PORT=${REDIS_PORT:-6379}
|
|
||||||
NO_SCHEDULER=${NO_SCHEDULER:-false}
|
|
||||||
CRON_SCHEDULE=${CRON_SCHEDULE:-'* 1 * * * *'}
|
|
||||||
|
|
||||||
NO_ARG_VALUE_ARGS=""
|
|
||||||
if [ "$NO_SCHEDULER" = "true" ]; then
|
|
||||||
NO_ARG_VALUE_ARGS+=(" --no-scheduler")
|
|
||||||
fi
|
|
||||||
|
|
||||||
set -x
|
|
||||||
|
|
||||||
export RUST_LOG=$RUST_LOG
|
|
||||||
exec /app/rsslair --port $PORT --cron "$CRON_SCHEDULE" $NO_ARG_VALUE_ARGS
|
|
||||||
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
-- definitions.lua
|
|
||||||
--- @diagnostic disable: unused-local
|
|
||||||
--- @diagnostic disable: missing-return
|
|
||||||
--- @diagnostic disable: lowercase-global
|
|
||||||
|
|
||||||
router = {}
|
|
||||||
|
|
||||||
--- @param name string
|
|
||||||
--- @param path string
|
|
||||||
function router.add_route(name, path) end
|
|
||||||
|
|
||||||
--- @param url string
|
|
||||||
--- @return string
|
|
||||||
function get(url) end
|
|
||||||
|
|
||||||
--- @class log
|
|
||||||
--- @field info fun(msg: string)
|
|
||||||
--- @field error fun(msg: string)
|
|
||||||
--- @field warn fun(msg: string)
|
|
||||||
--- @field debug fun(msg: string)
|
|
||||||
log = {}
|
|
||||||
|
|
||||||
--- @class HtmlParser
|
|
||||||
--- @field parse fun(html: string)
|
|
||||||
HtmlParser = {}
|
|
||||||
|
|
||||||
--- @class Feed
|
|
||||||
--- @field new fun(xml: string): Feed
|
|
||||||
--- @field render fun(): string
|
|
||||||
--- @field channel Channel
|
|
||||||
Feed = {}
|
|
||||||
|
|
||||||
--- @class Channel
|
|
||||||
--- @field title string
|
|
||||||
--- @field atom_link AtomLink
|
|
||||||
--- @field articles Article[]
|
|
||||||
Channel = {}
|
|
||||||
|
|
||||||
--- @class Article
|
|
||||||
--- @field title string
|
|
||||||
--- @field link string
|
|
||||||
--- @field creator string
|
|
||||||
--- @field pub_date string
|
|
||||||
--- @field categories string[]
|
|
||||||
--- @field guid Guid
|
|
||||||
--- @field description string
|
|
||||||
Article = {}
|
|
||||||
|
|
||||||
--- @class AtomLink
|
|
||||||
--- @field href string
|
|
||||||
--- @field rel string
|
|
||||||
--- @field type string
|
|
||||||
AtomLink = {}
|
|
||||||
|
|
||||||
--- @class Guid
|
|
||||||
--- @field is_perma_link boolean
|
|
||||||
--- @field guid string
|
|
||||||
Guid = {}
|
|
||||||
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
local WEBSITE_NAME = "TechCrunch"
|
|
||||||
local WEBSITE_HOME = "https://techcrunch.com"
|
|
||||||
|
|
||||||
add_route("techCrunch", "/TechCrunch")
|
|
||||||
|
|
||||||
techCrunch = {}
|
|
||||||
function techCrunch.route(args)
|
|
||||||
local xml = get("http://localhost:8081/feed.xml") -- Get an xml RSS feed
|
|
||||||
local rss_parser = Feed() -- Create a new instance of the Feed object
|
|
||||||
local feed = rss_parser:new(xml) -- Parse the xml into a feed object
|
|
||||||
|
|
||||||
sleep(1000)
|
|
||||||
|
|
||||||
local articles = feed.channel.articles -- Get all of the article objects
|
|
||||||
local article = articles[1] -- Get the first article object
|
|
||||||
log:info("Title: " .. article.title)
|
|
||||||
log:info("Description: " .. article.description)
|
|
||||||
|
|
||||||
local article_content = get(article.link) -- Get the entire article content
|
|
||||||
local html_parser = HtmlParser() -- Create a new instance of the html parser
|
|
||||||
html_parser:parse(article_content) -- Parse the article into an html tree
|
|
||||||
|
|
||||||
local elements = html_parser:select_element('.wp-block-post-content') -- Select the element with the class 'wp-block-post-content'
|
|
||||||
local element = elements[1] -- String of the html from the element selected
|
|
||||||
article.description = element -- Replace the description with the entire article
|
|
||||||
|
|
||||||
return feed:render(), feed
|
|
||||||
end
|
|
||||||
|
|
@ -0,0 +1,45 @@
|
||||||
|
add_route("Nineto5google", "/9to5google")
|
||||||
|
function Nineto5google.route(args)
|
||||||
|
-- TODO: This should not be handled by lua
|
||||||
|
-- Golang needs to bind the database to lua
|
||||||
|
local rssFeed = rss:get("https://9to5google.com/feed/")
|
||||||
|
local entries = parse_xml_feed(rssFeed)
|
||||||
|
|
||||||
|
local newEntries = {}
|
||||||
|
local to_scrape = {}
|
||||||
|
|
||||||
|
-- TODO: Potentially remove this limit
|
||||||
|
for i = 1, 25 do
|
||||||
|
table.insert(to_scrape, entries[i])
|
||||||
|
end
|
||||||
|
|
||||||
|
-- Check if the selected entries have already been scraped
|
||||||
|
scraped, to_scrape = db:check(to_scrape)
|
||||||
|
for _, entry in ipairs(to_scrape) do
|
||||||
|
log.debug("Scraping: " .. entry:link())
|
||||||
|
local article = get(entry:link())
|
||||||
|
local post = html.new(article)
|
||||||
|
post:remove("header")
|
||||||
|
post:remove("script")
|
||||||
|
post:remove(".ad-disclaimer-container")
|
||||||
|
post:remove("#after_disclaimer_placement")
|
||||||
|
post:remove(".adsbygoogle")
|
||||||
|
post:remove(".google-news-link")
|
||||||
|
|
||||||
|
local content = post:select(".post-content")
|
||||||
|
entry:description(content)
|
||||||
|
|
||||||
|
db:insert(entry)
|
||||||
|
table.insert(newEntries, entry)
|
||||||
|
|
||||||
|
os.execute("sleep 0.25")
|
||||||
|
end
|
||||||
|
-- Fetch the scraped entries from the database
|
||||||
|
local localEntries = db:getRss(scraped)
|
||||||
|
-- Merge the two lists
|
||||||
|
newEntries = rss:merge(localEntries, newEntries)
|
||||||
|
-- Create a new rss feed from the merged list
|
||||||
|
local image = RssImage.new("9to5google", "https://9to5google.com/favicon.ico", "https://9to5google.com")
|
||||||
|
local feed = create_rss_feed("9to5google", "https://9to5google.com", image, newEntries)
|
||||||
|
return feed
|
||||||
|
end
|
||||||
|
|
@ -2,45 +2,43 @@ local WEBSITE_NAME = "9to5mac"
|
||||||
local WEBSITE_HOME = "https://9to5mac.com"
|
local WEBSITE_HOME = "https://9to5mac.com"
|
||||||
|
|
||||||
add_route("Nineto5mac", "/9to5mac")
|
add_route("Nineto5mac", "/9to5mac")
|
||||||
Nineto5mac = {}
|
|
||||||
function Nineto5mac.route(args)
|
function Nineto5mac.route(args)
|
||||||
local xml = get("https://9to5mac.com/feed/")
|
local rssFeed = rss:get("https://9to5mac.com/feed/")
|
||||||
local rss_parser = Feed()
|
local entries = parse_xml_feed(rssFeed)
|
||||||
local feed = rss_parser:new(xml)
|
|
||||||
|
|
||||||
local articles = feed.channel.articles
|
local newEntries = {}
|
||||||
local existing_articles = db:check_if_articles_in_feed_exist(feed)
|
local to_scrape = {}
|
||||||
|
local scraped = {}
|
||||||
|
|
||||||
log:debug("Fetching missing articles from the database")
|
-- NOTE: 25 limit for right now
|
||||||
for _, article in ipairs(articles) do
|
to_scrape = rss.limit(entries, 25)
|
||||||
if existing_articles[article.guid.value] then
|
-- Check if the selected entries have already been scraped
|
||||||
log:debug("Article already exists in the database: " .. article.title)
|
scraped, to_scrape = db:check(to_scrape)
|
||||||
article.description = existing_articles[article.guid.value]
|
for _, entry in ipairs(to_scrape) do
|
||||||
goto continue
|
log.debug("Scraping: " .. entry:link())
|
||||||
end
|
local article = get(entry:link())
|
||||||
|
local post = html.new(article)
|
||||||
|
post:remove("header")
|
||||||
|
post:remove("script")
|
||||||
|
post:remove(".ad-disclaimer-container")
|
||||||
|
post:remove("#after_disclaimer_placement")
|
||||||
|
post:remove(".adsbygoogle")
|
||||||
|
post:remove(".google-news-link")
|
||||||
|
|
||||||
log:debug("Getting article: " .. article.title .. " from " .. article.link)
|
local content = post:select(".post-content")
|
||||||
|
entry:description(content)
|
||||||
|
|
||||||
local article_content = get(article.link)
|
db:insert(entry)
|
||||||
local html_parser = HtmlParser()
|
table.insert(newEntries, entry)
|
||||||
html_parser:parse(article_content)
|
|
||||||
|
|
||||||
-- Remove sections
|
os.execute("sleep 0.25")
|
||||||
html_parser:delete_element("header")
|
|
||||||
html_parser:delete_element("script")
|
|
||||||
html_parser:delete_element(".ad-disclaimer-container")
|
|
||||||
html_parser:delete_element("#after_disclaimer_placement")
|
|
||||||
html_parser:delete_element(".adsbygoogle")
|
|
||||||
html_parser:delete_element(".google-news-link")
|
|
||||||
|
|
||||||
local elements = html_parser:select_element('.post-content')
|
|
||||||
local element = elements
|
|
||||||
[1]
|
|
||||||
article.description =
|
|
||||||
element
|
|
||||||
sleep(500)
|
|
||||||
::continue::
|
|
||||||
end
|
end
|
||||||
|
-- Fetch the scraped entries from the database
|
||||||
return feed:render(), feed
|
local localEntries = db:getRss(scraped)
|
||||||
|
-- Merge the two lists
|
||||||
|
newEntries = rss.merge(localEntries, newEntries)
|
||||||
|
-- Create a new rss feed from the merged list
|
||||||
|
local image = RssImage.new(WEBSITE_NAME, "https://9to5mac.com/favicon.ico", WEBSITE_HOME)
|
||||||
|
local feed = create_rss_feed(WEBSITE_NAME, WEBSITE_HOME, image, newEntries)
|
||||||
|
return feed
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,45 @@
|
||||||
|
add_route("electrek", "/electrek")
|
||||||
|
function electrek.route(args)
|
||||||
|
-- TODO: This should not be handled by lua
|
||||||
|
-- Golang needs to bind the database to lua
|
||||||
|
local rssFeed = rss:get("https://electrek.co/feed")
|
||||||
|
local entries = parse_xml_feed(rssFeed)
|
||||||
|
|
||||||
|
local newEntries = {}
|
||||||
|
local to_scrape = {}
|
||||||
|
|
||||||
|
-- TODO: Potentially remove this limit
|
||||||
|
for i = 1, 25 do
|
||||||
|
table.insert(to_scrape, entries[i])
|
||||||
|
end
|
||||||
|
|
||||||
|
-- Check if the selected entries have already been scraped
|
||||||
|
scraped, to_scrape = db:check(to_scrape)
|
||||||
|
for _, entry in ipairs(to_scrape) do
|
||||||
|
log.debug("Scraping: " .. entry:link())
|
||||||
|
local article = get(entry:link())
|
||||||
|
local post = html.new(article)
|
||||||
|
post:remove("header")
|
||||||
|
post:remove("script")
|
||||||
|
post:remove(".ad-disclaimer-container")
|
||||||
|
post:remove("#after_disclaimer_placement")
|
||||||
|
post:remove(".adsbygoogle")
|
||||||
|
post:remove(".google-news-link")
|
||||||
|
|
||||||
|
local content = post:select(".post-content")
|
||||||
|
entry:description(content)
|
||||||
|
|
||||||
|
db:insert(entry)
|
||||||
|
table.insert(newEntries, entry)
|
||||||
|
|
||||||
|
os.execute("sleep 0.25")
|
||||||
|
end
|
||||||
|
-- Fetch the scraped entries from the database
|
||||||
|
local localEntries = db:getRss(scraped)
|
||||||
|
-- Merge the two lists
|
||||||
|
newEntries = rss:merge(localEntries, newEntries)
|
||||||
|
-- Create a new rss feed from the merged list
|
||||||
|
local image = RssImage.new("Electrek", "https://electrek.co/favicon.ico", "https://electrek.co")
|
||||||
|
local feed = create_rss_feed("Electrek", "https://electrek.co", image, newEntries)
|
||||||
|
return feed
|
||||||
|
end
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
-- TODO: Add the other routes here
|
||||||
|
add_route("eurogamer", "/eurogamer")
|
||||||
|
function eurogamer.route(args)
|
||||||
|
-- TODO: This should not be handled by lua
|
||||||
|
-- Golang needs to bind the database to lua
|
||||||
|
local rssFeed = rss:get("https://www.eurogamer.net/feed/news")
|
||||||
|
local entries = parse_xml_feed(rssFeed)
|
||||||
|
|
||||||
|
local newEntries = {}
|
||||||
|
local scraped = {}
|
||||||
|
local to_scrape = rss.limit(entries, 25)
|
||||||
|
|
||||||
|
-- Check if the selected entries have already been scraped
|
||||||
|
scraped, to_scrape = db:check(entries)
|
||||||
|
for _, entry in ipairs(to_scrape) do
|
||||||
|
log.debug("Scraping: " .. entry:link())
|
||||||
|
local article = get(entry:link())
|
||||||
|
local post = html.new(article)
|
||||||
|
post:remove("header")
|
||||||
|
post:remove("script")
|
||||||
|
post:remove(".poll_wrapper")
|
||||||
|
post:remove(".poll_container")
|
||||||
|
post:remove(".poll")
|
||||||
|
post:remove(".poll_leaderboard")
|
||||||
|
post:remove(".advert_container")
|
||||||
|
post:remove(".article_footer")
|
||||||
|
post:remove(".adsbygoogle")
|
||||||
|
post:remove(".google-news-link")
|
||||||
|
|
||||||
|
local content = post:select(".article_body")
|
||||||
|
entry:description(content)
|
||||||
|
|
||||||
|
db:insert(entry)
|
||||||
|
table.insert(newEntries, entry)
|
||||||
|
|
||||||
|
os.execute("sleep 0.25")
|
||||||
|
end
|
||||||
|
-- Fetch the scraped entries from the database
|
||||||
|
local localEntries = db:getRss(scraped)
|
||||||
|
|
||||||
|
-- Merge the two lists
|
||||||
|
newEntries = rss.merge(newEntries, localEntries)
|
||||||
|
-- Create a new rss feed from the merged list
|
||||||
|
local image = RssImage.new("Eurogamer News", "https://eurogamer.net/favicon.ico", "https://eurogamer.net")
|
||||||
|
local feed = create_rss_feed("Eurogamer News", "https://eurogamer.net", image, newEntries)
|
||||||
|
return feed
|
||||||
|
end
|
||||||
|
|
@ -1,39 +1,37 @@
|
||||||
local WEBSITE_NAME = "Godot"
|
|
||||||
local WEBSITE_HOME = "https://godotengine.org"
|
|
||||||
|
|
||||||
add_route('godot', '/godot')
|
add_route('godot', '/godot')
|
||||||
godot = {}
|
|
||||||
|
-- Live Website
|
||||||
|
-- THIS WORKS!!!
|
||||||
function godot.route(args)
|
function godot.route(args)
|
||||||
local xml = get("https://godotengine.org/rss.xml")
|
-- Get the RSS Feed
|
||||||
local rss_parser = Feed()
|
local rssFeed = rss:get("https://godotengine.org/rss.xml")
|
||||||
local feed = rss_parser:new(xml)
|
local entries = parse_xml_feed(rssFeed)
|
||||||
|
local newEntries = {}
|
||||||
|
local scraped, to_scrape = db:check(entries)
|
||||||
|
|
||||||
local articles = feed.channel.articles
|
-- Accuumulate the entries
|
||||||
local existing_articles = db:check_if_articles_in_feed_exist(feed)
|
for _, entry in ipairs(to_scrape) do
|
||||||
|
log.debug("Scraping: " .. entry:link())
|
||||||
log:debug("Fetching missing articles from the database")
|
-- Get the article
|
||||||
for _, article in ipairs(articles) do
|
local article = get(entry:link())
|
||||||
if existing_articles[article.guid.value] then
|
local article_html = html.new(article)
|
||||||
log:debug("Article already exists in the database: " .. article.title)
|
-- Get the article body
|
||||||
article.description = existing_articles[article.guid.value]
|
local content = article_html:select("div.article-body")
|
||||||
goto continue
|
-- Update the entry
|
||||||
end
|
entry:description(content)
|
||||||
|
-- Add the entry to the list
|
||||||
log:debug("Getting article: " .. article.title .. " from " .. article.link)
|
db:insert(entry)
|
||||||
|
table.insert(newEntries, entry)
|
||||||
local article_content = get(article.link)
|
os.execute("sleep 0.25")
|
||||||
local html_parser = HtmlParser()
|
|
||||||
html_parser:parse(article_content)
|
|
||||||
|
|
||||||
local elements = html_parser:select_element('.article-body')
|
|
||||||
local element = elements
|
|
||||||
[1]
|
|
||||||
article.description =
|
|
||||||
element
|
|
||||||
sleep(500)
|
|
||||||
::continue::
|
|
||||||
end
|
end
|
||||||
|
|
||||||
return feed:render(), feed
|
-- Get the already scraped entries from the database
|
||||||
end
|
local localEntries = db:getRss(scraped)
|
||||||
|
-- TODO: Handle case where there are no local entries
|
||||||
|
local newFeed = rss.merge(localEntries, newEntries)
|
||||||
|
|
||||||
|
-- Create a new rss feed
|
||||||
|
local image = RssImage.new("Godot Engine Official", "https://godotengine.org/favicon.ico", "https://godotengine.org")
|
||||||
|
local feed = create_rss_feed("Godot Engine Official", "https://godotengine.org", image, newFeed)
|
||||||
|
return feed
|
||||||
|
end
|
||||||
|
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
-- TODO: Once the feed is sufficently long, we could make a bigger feed since the feed is only 20 articles long
|
|
||||||
local WEBSITE_NAME = "Phronix"
|
|
||||||
local WEBSITE_HOME = "https://phoronix.com"
|
|
||||||
|
|
||||||
add_route("phoronix", "/phoronix")
|
|
||||||
|
|
||||||
phoronix = {}
|
|
||||||
function phoronix.route(args)
|
|
||||||
local xml = get("https://www.phoronix.com/rss.php") -- Get the xml from the website
|
|
||||||
local rss_parser = Feed() -- Create a new instance of the Feed object
|
|
||||||
local feed = rss_parser:new(xml) -- Parse the xml into a feed object
|
|
||||||
|
|
||||||
local articles = feed.channel.articles -- Get all of the article objects
|
|
||||||
|
|
||||||
-- TODO: Add api to check if the articles are already in the database
|
|
||||||
local existing_articles = db:check_if_articles_in_feed_exist(feed) -- Get the missing articles from the database
|
|
||||||
|
|
||||||
log:debug("Fetching missing articles from the database")
|
|
||||||
for _, article in ipairs(articles) do
|
|
||||||
if existing_articles[article.guid.value] then
|
|
||||||
log:debug("Article already exists in the database: " .. article.title)
|
|
||||||
article.description = existing_articles[article.guid.value]
|
|
||||||
goto continue
|
|
||||||
end
|
|
||||||
|
|
||||||
log:debug("Getting article: " .. article.title .. " from " .. article.link)
|
|
||||||
|
|
||||||
local article_content = get(article.link) -- Get the entire article content
|
|
||||||
local html_parser = HtmlParser() -- Create a new instance of the html parser
|
|
||||||
html_parser:parse(article_content) -- Parse the article into an html tree
|
|
||||||
|
|
||||||
local elements = html_parser:select_element('.content') -- Select the element with the class 'wp-block-post-content'
|
|
||||||
local element = elements
|
|
||||||
[1] -- String of the html from the element selected
|
|
||||||
article.description =
|
|
||||||
element -- Replace the description with the entire article
|
|
||||||
sleep(500)
|
|
||||||
::continue::
|
|
||||||
end
|
|
||||||
|
|
||||||
return feed:render(), feed
|
|
||||||
end
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
-- TODO: Once the feed is sufficently long, we could make a bigger feed since the feed is only 20 articles long
|
|
||||||
local WEBSITE_NAME = "TechCrunch"
|
|
||||||
local WEBSITE_HOME = "https://techcrunch.com"
|
|
||||||
|
|
||||||
add_route("techCrunch", "/TechCrunch")
|
|
||||||
|
|
||||||
techCrunch = {}
|
|
||||||
function techCrunch.route(args)
|
|
||||||
local xml = get("https://techcrunch.com/feed/") -- Get the xml from the website
|
|
||||||
local rss_parser = Feed() -- Create a new instance of the Feed object
|
|
||||||
local feed = rss_parser:new(xml) -- Parse the xml into a feed object
|
|
||||||
|
|
||||||
local articles = feed.channel.articles -- Get all of the article objects
|
|
||||||
|
|
||||||
-- TODO: Add api to check if the articles are already in the database
|
|
||||||
local existing_articles = db:check_if_articles_in_feed_exist(feed) -- Get the missing articles from the database
|
|
||||||
|
|
||||||
log:debug("Fetching missing articles from the database")
|
|
||||||
for _, article in ipairs(articles) do
|
|
||||||
if existing_articles[article.guid.value] then
|
|
||||||
log:debug("Article already exists in the database: " .. article.title)
|
|
||||||
article.description = existing_articles[article.guid.value]
|
|
||||||
goto continue
|
|
||||||
end
|
|
||||||
|
|
||||||
log:debug("Getting article: " .. article.title .. " from " .. article.link)
|
|
||||||
|
|
||||||
local article_content = get(article.link) -- Get the entire article content
|
|
||||||
local html_parser = HtmlParser() -- Create a new instance of the html parser
|
|
||||||
html_parser:parse(article_content) -- Parse the article into an html tree
|
|
||||||
|
|
||||||
local elements = html_parser:select_element('.wp-block-post-content') -- Select the element with the class 'wp-block-post-content'
|
|
||||||
local element = elements
|
|
||||||
[1] -- String of the html from the element selected
|
|
||||||
article.description =
|
|
||||||
element -- Replace the description with the entire article
|
|
||||||
sleep(500)
|
|
||||||
::continue::
|
|
||||||
end
|
|
||||||
|
|
||||||
return feed:render(), feed
|
|
||||||
end
|
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
-- @meta
|
||||||
|
---@diagnostic disable: unused-local
|
||||||
|
---@diagnostic disable: lowercase-global
|
||||||
|
|
||||||
|
-- @param name string
|
||||||
|
-- @param path string
|
||||||
|
-- return void
|
||||||
|
function add_route(name, path) end
|
||||||
|
|
||||||
|
-- @param url string
|
||||||
|
-- @return string
|
||||||
|
function get(url) end
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
lua "github.com/yuin/gopher-lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
func start_server(address string, port string, L *lua.LState) *http.Server {
|
||||||
|
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
path := r.URL.Path
|
||||||
|
logger.Debug("Request received: ", path)
|
||||||
|
|
||||||
|
if _, ok := routes[path]; !ok {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
response := responses[path]
|
||||||
|
logger.Trace("API Response: ", response)
|
||||||
|
w.Write([]byte(response))
|
||||||
|
})
|
||||||
|
|
||||||
|
listening := address + ":" + port
|
||||||
|
server := &http.Server{
|
||||||
|
Addr: listening,
|
||||||
|
Handler: nil,
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
// TODO: Add an error message for this
|
||||||
|
if err := server.ListenAndServe(); err != http.ErrServerClosed {
|
||||||
|
logger.Fatalf("Server Error: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
logger.Info("Listening on ", listening)
|
||||||
|
return server
|
||||||
|
}
|
||||||
280
src/database.rs
280
src/database.rs
|
|
@ -1,280 +0,0 @@
|
||||||
use crate::rss_parser::{Item as Article, Rss};
|
|
||||||
use log::{debug, error, trace};
|
|
||||||
use mlua::{FromLua, Lua, Table, UserData, UserDataMethods, Value};
|
|
||||||
use once_cell::sync::Lazy;
|
|
||||||
use rusqlite::{params, Connection};
|
|
||||||
use std::sync::Mutex;
|
|
||||||
|
|
||||||
static SQLITE_DB: Lazy<Mutex<Connection>> =
|
|
||||||
Lazy::new(|| Mutex::new(Connection::open("database.db").unwrap()));
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct Database {
|
|
||||||
conn: &'static Mutex<Connection>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for Database {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Database {
|
|
||||||
pub fn new() -> Database {
|
|
||||||
//TODO: Handle all of these unwrap calls
|
|
||||||
let conn = SQLITE_DB.lock().unwrap();
|
|
||||||
conn.execute(
|
|
||||||
"CREATE TABLE IF NOT EXISTS feeds (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
url TEXT NOT NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
last_updated TEXT NOT NULL
|
|
||||||
)",
|
|
||||||
params![],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
conn.execute(
|
|
||||||
"CREATE TABLE IF NOT EXISTS images (
|
|
||||||
url TEXT PRIMARY KEY,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
link TEXT NOT NULL,
|
|
||||||
width INTEGER NOT NULL,
|
|
||||||
height INTEGER NOT NULL
|
|
||||||
)",
|
|
||||||
params![],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
conn.execute(
|
|
||||||
"CREATE TABLE IF NOT EXISTS articles (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
guid TEXT UNIQUE NOT NULL,
|
|
||||||
creator TEXT,
|
|
||||||
categories TEXT,
|
|
||||||
feed_id INTEGER NOT NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
link TEXT NOT NULL,
|
|
||||||
pub_date TEXT NOT NULL,
|
|
||||||
description TEXT NOT NULL
|
|
||||||
)",
|
|
||||||
params![],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
Self { conn: &*SQLITE_DB }
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_with_conn(conn: Connection) -> Database {
|
|
||||||
conn.execute(
|
|
||||||
"CREATE TABLE IF NOT EXISTS feeds (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
url TEXT NOT NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
last_updated TEXT NOT NULL
|
|
||||||
)",
|
|
||||||
params![],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
conn.execute(
|
|
||||||
"CREATE TABLE IF NOT EXISTS images (
|
|
||||||
url TEXT PRIMARY KEY,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
link TEXT NOT NULL,
|
|
||||||
width INTEGER NOT NULL,
|
|
||||||
height INTEGER NOT NULL
|
|
||||||
)",
|
|
||||||
params![],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
conn.execute(
|
|
||||||
"CREATE TABLE IF NOT EXISTS articles (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
guid TEXT UNIQUE NOT NULL,
|
|
||||||
creator TEXT,
|
|
||||||
categories TEXT,
|
|
||||||
feed_id INTEGER NOT NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
link TEXT NOT NULL,
|
|
||||||
pub_date TEXT NOT NULL,
|
|
||||||
description TEXT NOT NULL
|
|
||||||
)",
|
|
||||||
params![],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
Self { conn: &*SQLITE_DB }
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn insert_feed(
|
|
||||||
&self,
|
|
||||||
url: &str,
|
|
||||||
title: &str,
|
|
||||||
last_updated: &str,
|
|
||||||
) -> Result<usize, Box<dyn std::error::Error>> {
|
|
||||||
let conn = self.conn.lock().unwrap();
|
|
||||||
Ok(conn.execute(
|
|
||||||
"INSERT INTO feeds (url, title, last_updated) VALUES (?1, ?2, ?3)",
|
|
||||||
params![url, title, last_updated],
|
|
||||||
)?)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Insert an article into the database
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `article` - The article to insert
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
///
|
|
||||||
/// * The number of rows inserted
|
|
||||||
pub fn insert_article(&self, article: &Article) -> Result<usize, Box<dyn std::error::Error>> {
|
|
||||||
let title = article.title.as_ref().unwrap();
|
|
||||||
let guid = article.guid.as_ref().unwrap().value.clone();
|
|
||||||
|
|
||||||
let conn = self.conn.lock().unwrap();
|
|
||||||
let ret = conn.execute(
|
|
||||||
"INSERT INTO articles (guid, creator, categories, feed_id, title, link, pub_date, description) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
|
||||||
params![
|
|
||||||
guid,
|
|
||||||
article.creator,
|
|
||||||
article.categories.join(","),
|
|
||||||
1,
|
|
||||||
title,
|
|
||||||
article.link,
|
|
||||||
article.pub_date,
|
|
||||||
article.description,
|
|
||||||
],
|
|
||||||
);
|
|
||||||
match ret {
|
|
||||||
Ok(_) => Ok(ret?),
|
|
||||||
Err(_) => {
|
|
||||||
// TODO: Do we care about entries that already exist?
|
|
||||||
//error!("Error inserting {} into database: {}", title, e);
|
|
||||||
//Err(e.into())
|
|
||||||
|
|
||||||
Ok(0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_article_by_guid(&self, guid: &str) -> mlua::Result<Option<Article>> {
|
|
||||||
//TODO: Make this look better
|
|
||||||
let conn = self.conn.lock().unwrap();
|
|
||||||
let mut stmt = match conn.prepare("SELECT * FROM articles WHERE guid = ?1") {
|
|
||||||
Ok(it) => it,
|
|
||||||
Err(err) => {
|
|
||||||
let var_name = Err(err);
|
|
||||||
return var_name.unwrap();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let mut rows = match stmt.query(params![guid]) {
|
|
||||||
Ok(it) => it,
|
|
||||||
Err(err) => {
|
|
||||||
let var_name = Err(err);
|
|
||||||
return var_name.unwrap();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let row = match rows.next() {
|
|
||||||
Ok(it) => it,
|
|
||||||
Err(err) => {
|
|
||||||
let var_name = Err(err);
|
|
||||||
return var_name.unwrap();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let row = match row {
|
|
||||||
Some(it) => it,
|
|
||||||
None => return Ok(None),
|
|
||||||
};
|
|
||||||
trace!("Row: {:?}", row);
|
|
||||||
let article = Article {
|
|
||||||
title: Some(row.get(5).unwrap_or("".to_string())),
|
|
||||||
guid: Some(crate::rss_parser::Guid {
|
|
||||||
value: Some(row.get(1).unwrap_or("".to_string())),
|
|
||||||
is_perma_link: Some(false),
|
|
||||||
}),
|
|
||||||
creator: Some(row.get(2).unwrap_or("".to_string())),
|
|
||||||
categories: row
|
|
||||||
.get(3)
|
|
||||||
.unwrap_or("".to_string())
|
|
||||||
.split(",")
|
|
||||||
.map(|s| s.to_string())
|
|
||||||
.collect(),
|
|
||||||
link: Some(row.get(6).unwrap_or("".to_string())),
|
|
||||||
pub_date: Some(row.get(7).unwrap_or("".to_string())),
|
|
||||||
description: Some(row.get(8).unwrap_or("".to_string())),
|
|
||||||
};
|
|
||||||
debug!("Article: {:?}", article);
|
|
||||||
|
|
||||||
Ok(Some(article))
|
|
||||||
}
|
|
||||||
|
|
||||||
//pub fn does_article_exist(&self, guid: &str) -> mlua::Result<bool> {
|
|
||||||
// let mut stmt = self.conn.prepare("SELECT * FROM articles WHERE guid = ?1")?;
|
|
||||||
// let mut rows = stmt.query(params![guid])?;
|
|
||||||
// let row = rows.next()?;
|
|
||||||
// match row {
|
|
||||||
// Some(_) => Ok(true),
|
|
||||||
// None => Ok(false),
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FromLua<'_> for Database {
|
|
||||||
fn from_lua(value: Value, _: &'_ Lua) -> mlua::Result<Self> {
|
|
||||||
match value {
|
|
||||||
Value::UserData(ud) => {
|
|
||||||
let db = ud.borrow::<Database>()?;
|
|
||||||
Ok(db.clone())
|
|
||||||
}
|
|
||||||
_ => Err(mlua::Error::FromLuaConversionError {
|
|
||||||
from: value.type_name(),
|
|
||||||
to: "Database",
|
|
||||||
message: Some("expected Database".to_string()),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl UserData for Database {
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method(
|
|
||||||
"get_article_by_guid",
|
|
||||||
|_, this, guid: String| -> Result<Article, mlua::Error> {
|
|
||||||
match this.get_article_by_guid(&guid) {
|
|
||||||
Ok(Some(article)) => Ok(article),
|
|
||||||
Ok(None) => Err(mlua::Error::RuntimeError("Article not found".to_string())),
|
|
||||||
Err(e) => Err(mlua::Error::RuntimeError(e.to_string())),
|
|
||||||
}
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
methods.add_method(
|
|
||||||
"check_if_articles_in_feed_exist",
|
|
||||||
|lua, this, feed: Rss| -> Result<Table, mlua::Error> {
|
|
||||||
// Check to if all of the articles in the feed exist
|
|
||||||
let table = lua.create_table()?;
|
|
||||||
|
|
||||||
let articles = &feed.channel.borrow().items;
|
|
||||||
for article in articles {
|
|
||||||
let article = article.borrow();
|
|
||||||
let guid = &article.guid.as_ref().unwrap().value;
|
|
||||||
let guid = guid.clone().unwrap();
|
|
||||||
match this.get_article_by_guid(&guid) {
|
|
||||||
Ok(Some(article)) => {
|
|
||||||
table.set(guid, article.description.clone().unwrap())?;
|
|
||||||
}
|
|
||||||
Ok(None) => {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!("Error checking if article exists: {}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(table)
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
//methods.add_method("does_article_exist", |_, this, guid: String| -> Result<Result<bool, _>, mlua::Error> {
|
|
||||||
// Ok(this.does_article_exist(&guid))
|
|
||||||
//});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
use log::{debug, error};
|
|
||||||
use mlua::{UserData, UserDataMethods};
|
|
||||||
|
|
||||||
use scraper::{Html, Selector};
|
|
||||||
|
|
||||||
pub struct HtmlParser {
|
|
||||||
doc: Html,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for HtmlParser {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl HtmlParser {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
doc: Html::parse_document(""),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn parse(&mut self, html: &str) {
|
|
||||||
self.doc = Html::parse_document(html);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn select_element(&self, selector: &str) -> Vec<String> {
|
|
||||||
let selector = Selector::parse(selector);
|
|
||||||
match selector {
|
|
||||||
Ok(selector) => {
|
|
||||||
let elements: Vec<_> = self.doc.select(&selector).collect();
|
|
||||||
debug!("Found {} elements", elements.len());
|
|
||||||
elements.iter().map(|x| x.html()).collect()
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!("{}", e);
|
|
||||||
vec![]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn delete_element(&mut self, selector: &str) {
|
|
||||||
let selector = Selector::parse(selector);
|
|
||||||
match selector {
|
|
||||||
Ok(selector) => {
|
|
||||||
let node_ids: Vec<_> = self.doc.select(&selector).map(|x| x.id()).collect();
|
|
||||||
for id in node_ids {
|
|
||||||
debug!("Deleting node with id: {:?}", id);
|
|
||||||
self.doc.tree.get_mut(id).unwrap().detach();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
println!("Error: {}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_html(&self) -> String {
|
|
||||||
self.doc.root_element().html()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl UserData for HtmlParser {
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method_mut(
|
|
||||||
"parse",
|
|
||||||
|_, this, html: String| -> Result<(), mlua::Error> {
|
|
||||||
this.parse(&html);
|
|
||||||
Ok(())
|
|
||||||
},
|
|
||||||
);
|
|
||||||
methods.add_method(
|
|
||||||
"select_element",
|
|
||||||
|_, this, selector: String| -> Result<Vec<String>, mlua::Error> {
|
|
||||||
Ok(this.select_element(&selector))
|
|
||||||
},
|
|
||||||
);
|
|
||||||
methods.add_method_mut(
|
|
||||||
"delete_element",
|
|
||||||
|_, this, selector: String| -> Result<(), mlua::Error> {
|
|
||||||
this.delete_element(&selector);
|
|
||||||
Ok(())
|
|
||||||
},
|
|
||||||
);
|
|
||||||
methods.add_method("render", |_, this, _: ()| -> Result<String, mlua::Error> {
|
|
||||||
Ok(this.get_html())
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
91
src/main.rs
91
src/main.rs
|
|
@ -1,91 +0,0 @@
|
||||||
use log::{debug, info};
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::{Arc, Mutex, RwLock};
|
|
||||||
use std::thread;
|
|
||||||
use once_cell::sync::Lazy;
|
|
||||||
use clap::Parser;
|
|
||||||
|
|
||||||
pub mod html_parser;
|
|
||||||
pub mod router;
|
|
||||||
pub mod rss_parser;
|
|
||||||
pub mod scheduler;
|
|
||||||
pub mod scripting;
|
|
||||||
pub mod database;
|
|
||||||
|
|
||||||
#[derive(Parser)]
|
|
||||||
#[command(version, about, long_about = None)]
|
|
||||||
struct Cli {
|
|
||||||
#[arg(short, long, default_value = "8000")]
|
|
||||||
port: u16,
|
|
||||||
|
|
||||||
#[arg(long, default_value = "127.0.0.1")]
|
|
||||||
redis_url: String,
|
|
||||||
|
|
||||||
#[arg(short, long, default_value = "6379")]
|
|
||||||
redis_port: u16,
|
|
||||||
|
|
||||||
#[arg(short, long)]
|
|
||||||
no_scheduler: bool,
|
|
||||||
|
|
||||||
#[arg(short, long, default_value = "1m")]
|
|
||||||
cron: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
static REDIS: Lazy<Mutex<redis::Connection>> = Lazy::new(|| {
|
|
||||||
let client = redis::Client::open("redis://127.0.0.1").unwrap();
|
|
||||||
let conn = client.get_connection().unwrap();
|
|
||||||
Mutex::new(conn)
|
|
||||||
});
|
|
||||||
|
|
||||||
#[tokio::main]
|
|
||||||
async fn main() {
|
|
||||||
let cli = Cli::parse();
|
|
||||||
let port = cli.port;
|
|
||||||
let cron = cli.cron;
|
|
||||||
let no_scheduler = cli.no_scheduler;
|
|
||||||
|
|
||||||
env_logger::init();
|
|
||||||
|
|
||||||
let routes = Arc::new(RwLock::new(HashMap::new()));
|
|
||||||
let routes_ref = routes.clone();
|
|
||||||
|
|
||||||
// Create variable for when the routes are ready
|
|
||||||
let is_ready = Arc::new(Mutex::new(false));
|
|
||||||
let is_ready_ref = is_ready.clone();
|
|
||||||
|
|
||||||
thread::spawn(move || {
|
|
||||||
let engine = scripting::ScriptingEngine::new();
|
|
||||||
engine.load_globals().unwrap();
|
|
||||||
engine.load_all_scripts().unwrap();
|
|
||||||
|
|
||||||
let registered_routes = engine.get_routes().unwrap();
|
|
||||||
routes_ref.write().unwrap().extend(registered_routes);
|
|
||||||
std::mem::drop(routes_ref
|
|
||||||
.write()
|
|
||||||
.unwrap());
|
|
||||||
|
|
||||||
let mut r = is_ready_ref.lock().unwrap();
|
|
||||||
*r = true;
|
|
||||||
std::mem::drop(r); // Unlock the mutex
|
|
||||||
|
|
||||||
if no_scheduler {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut scheduler = scheduler::Scheduler::new(engine).unwrap(); //TODO: Handle error
|
|
||||||
//scheduler.schedule("* 1 * * * *").unwrap();
|
|
||||||
scheduler.schedule(&cron).unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let is_ready = is_ready.clone();
|
|
||||||
while !*is_ready.lock().unwrap() {
|
|
||||||
thread::sleep(std::time::Duration::from_secs(1));
|
|
||||||
}
|
|
||||||
debug!("Routes are ready");
|
|
||||||
|
|
||||||
let routes = routes.read().unwrap().clone();
|
|
||||||
let router = router::Router::new(routes.clone(), port);
|
|
||||||
info!("Successfully created router and scripting engine");
|
|
||||||
|
|
||||||
router.serve().await;
|
|
||||||
}
|
|
||||||
|
|
@ -1,93 +0,0 @@
|
||||||
use core::fmt;
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use anyhow::Result;
|
|
||||||
use log::debug;
|
|
||||||
use redis::{Commands, RedisResult};
|
|
||||||
use warp::http::StatusCode;
|
|
||||||
use warp::Filter;
|
|
||||||
|
|
||||||
use crate::REDIS;
|
|
||||||
|
|
||||||
pub struct Router {
|
|
||||||
//TODO: Redis to cache the route response
|
|
||||||
routes: HashMap<String, String>,
|
|
||||||
port: u16,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl fmt::Debug for Router {
|
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
||||||
write!(f, "Router")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Router {
|
|
||||||
pub fn new(routes: HashMap<String, String>, port: u16) -> Router {
|
|
||||||
debug!("Creating new router with routes: {:?}", routes);
|
|
||||||
|
|
||||||
Router { routes, port }
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_routes(&self) -> HashMap<String, String> {
|
|
||||||
self.routes.clone()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn serve(&self) {
|
|
||||||
async fn dyn_reply(
|
|
||||||
word: String,
|
|
||||||
routes: &HashMap<String, String>,
|
|
||||||
) -> Result<Box<dyn warp::Reply>, warp::Rejection> {
|
|
||||||
debug!("Received word: {}", word);
|
|
||||||
debug!("Routes: {:?}", routes);
|
|
||||||
let response: RedisResult<String> = REDIS.lock().unwrap().get(word.clone());
|
|
||||||
match response {
|
|
||||||
Ok(response) => {
|
|
||||||
debug!("Response from Redis: {}", response);
|
|
||||||
let resp = warp::http::Response::builder()
|
|
||||||
.header("Content-Type", "text/xml")
|
|
||||||
.status(StatusCode::OK)
|
|
||||||
.body(response.clone())
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
Ok(Box::new(warp::reply::with_status(resp, StatusCode::OK)))
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
debug!("Response not found in Redis, checking routes");
|
|
||||||
debug!("Route not found");
|
|
||||||
Ok(Box::new(warp::reply::with_status(
|
|
||||||
"Route not found".to_string(),
|
|
||||||
StatusCode::NOT_FOUND,
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
let r = self.routes.clone();
|
|
||||||
let r2 = self.routes.clone();
|
|
||||||
let index = warp::path::end().map(move || {
|
|
||||||
let mut html_str = "".to_string();
|
|
||||||
// Create table for the routes with borders
|
|
||||||
html_str.push_str("<style>table {border-collapse: collapse;} table, th, td {border: 1px solid black;}</style>");
|
|
||||||
html_str.push_str("<table>");
|
|
||||||
html_str.push_str("<tr><th>Service</th><th>Endpoint</th></tr>");
|
|
||||||
for (route, response) in r.iter() {
|
|
||||||
html_str.push_str("<tr>");
|
|
||||||
html_str.push_str(&format!("<td>{}</td>", route));
|
|
||||||
html_str.push_str(&format!("<td>{}</td>", response));
|
|
||||||
html_str.push_str("</tr>");
|
|
||||||
}
|
|
||||||
html_str.push_str("</table>");
|
|
||||||
|
|
||||||
warp::reply::html(html_str)
|
|
||||||
});
|
|
||||||
let routes = warp::path::param().and_then({
|
|
||||||
move |word: String| {
|
|
||||||
let s = r2.clone();
|
|
||||||
async move { dyn_reply(word, &s).await }
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.or(index);
|
|
||||||
warp::serve(routes).run(([0, 0, 0, 0], self.port)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,330 +0,0 @@
|
||||||
use std::{cell::RefCell, rc::Rc};
|
|
||||||
|
|
||||||
use log::error;
|
|
||||||
use mlua::{FromLua, Lua, Result, UserData, UserDataFields, UserDataMethods, Value};
|
|
||||||
use quick_xml::de::from_str;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
// TODO: Define macro to cleanup #[derive] boilerplate
|
|
||||||
|
|
||||||
// NOTE: Macros for lua userdata fields and methods
|
|
||||||
macro_rules! add_field_method_get {
|
|
||||||
($fields:ident, $name:literal, $field:ident) => {
|
|
||||||
$fields.add_field_method_get($name, |_, this| Ok(this.$field.clone()));
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_field_method_get_mut {
|
|
||||||
($fields:ident, $name:literal, $field:ident) => {
|
|
||||||
$fields.add_field_method_get($name, |_, this| Ok(this.$field));
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_field_method_set {
|
|
||||||
($fields:ident, $name:literal, $field:ident, $type:ty) => {
|
|
||||||
$fields.add_field_method_set($name, |_, this, value: $type| {
|
|
||||||
this.$field = value;
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_field_method_get_set {
|
|
||||||
($fields:ident, $name:literal, $field:ident, $type:ty) => {
|
|
||||||
add_field_method_get!($fields, $name, $field);
|
|
||||||
add_field_method_set!($fields, $name, $field, $type);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_field_method_get_rc {
|
|
||||||
($fields:ident, $name:literal, $field:ident) => {
|
|
||||||
$fields.add_field_method_get($name, |_, this| Ok(this.$field.clone()));
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_field_method_set_rc {
|
|
||||||
($fields:ident, $name:literal, $field:ident, $type:ty) => {
|
|
||||||
$fields.add_field_method_set($name, |_, this, value: $type| {
|
|
||||||
this.$field = Rc::new(RefCell::new(value));
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_field_method_get_set_rc {
|
|
||||||
($fields:ident, $name:literal, $field:ident, $type:ty) => {
|
|
||||||
add_field_method_get_rc!($fields, $name, $field);
|
|
||||||
add_field_method_set_rc!($fields, $name, $field, $type);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_field_method_set_rc_option {
|
|
||||||
($fields:ident, $name:literal, $field:ident, $type:ty) => {
|
|
||||||
$fields.add_field_method_set($name, |_, this, value: Option<$type>| {
|
|
||||||
this.$field = value.map(|v| Rc::new(RefCell::new(v)));
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! implement_from_lua {
|
|
||||||
($type:ty) => {
|
|
||||||
impl FromLua<'_> for $type {
|
|
||||||
fn from_lua(value: Value, _: &Lua) -> Result<Self> {
|
|
||||||
match value {
|
|
||||||
Value::UserData(ud) => Ok(ud.borrow::<Self>()?.clone()),
|
|
||||||
_ => unreachable!(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Lua object representing an RSS feed as Feed
|
|
||||||
#[derive(Default, Debug, Deserialize, Serialize, Clone)]
|
|
||||||
#[serde(rename = "rss")]
|
|
||||||
pub struct Rss {
|
|
||||||
#[serde(rename = "channel")]
|
|
||||||
pub channel: Rc<RefCell<Channel>>,
|
|
||||||
|
|
||||||
// Attributes
|
|
||||||
// Example: <rss version="0"/>
|
|
||||||
#[serde(rename = "@version")]
|
|
||||||
pub version: Option<String>,
|
|
||||||
#[serde(rename = "@xmlns:content")]
|
|
||||||
pub xmlns_content: Option<String>,
|
|
||||||
#[serde(rename = "@xmlns:wfw")]
|
|
||||||
pub xmlns_wfw: Option<String>,
|
|
||||||
#[serde(rename = "@xmlns:dc")]
|
|
||||||
pub xmlns_dc: Option<String>,
|
|
||||||
#[serde(rename = "@xmlns:atom")]
|
|
||||||
pub xmlns_atom: Option<String>,
|
|
||||||
#[serde(rename = "@xmlns:sy")]
|
|
||||||
pub xmlns_sy: Option<String>,
|
|
||||||
#[serde(rename = "@xmlns:slash")]
|
|
||||||
pub xmlns_slash: Option<String>,
|
|
||||||
#[serde(rename = "@xmlns:georss")]
|
|
||||||
pub xmlns_georss: Option<String>,
|
|
||||||
#[serde(rename = "@xmlns:geo")]
|
|
||||||
pub xmlns_geo: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Rss {
|
|
||||||
fn new(xml: &str) -> Rss {
|
|
||||||
let xml = xml.replace("<atom:link", "<atom_link");
|
|
||||||
from_str(&xml).unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn render(&self) -> String {
|
|
||||||
quick_xml::se::to_string(self)
|
|
||||||
.unwrap()
|
|
||||||
.replace("<atom_link>", "<atom:link>")
|
|
||||||
.replace("</atom_link>", "</atom:link>")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FromLua<'_> for Rss {
|
|
||||||
fn from_lua(value: Value, _: &Lua) -> Result<Self> {
|
|
||||||
match value {
|
|
||||||
Value::UserData(ud) => Ok(ud.borrow::<Self>()?.clone()),
|
|
||||||
_ => {
|
|
||||||
error!("Expected (string, Feed) tuple, received: {:?}", value);
|
|
||||||
error!("Maybe the route function is not returning both");
|
|
||||||
Err(mlua::Error::RuntimeError(
|
|
||||||
"Expected (string, Feed) tuple".to_string(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl UserData for Rss {
|
|
||||||
fn add_fields<'lua, F: UserDataFields<'lua, Self>>(fields: &mut F) {
|
|
||||||
add_field_method_get_set_rc!(fields, "channel", channel, Channel);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method("new", |_, _, xml: String| Ok(Rss::new(&xml)));
|
|
||||||
methods.add_method("render", |_, this, ()| Ok(this.render()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default, Debug, Deserialize, Serialize, Clone)]
|
|
||||||
pub struct Channel {
|
|
||||||
pub title: Option<String>,
|
|
||||||
#[serde(rename = "atom_link")]
|
|
||||||
pub atom_link: Option<Rc<RefCell<AtomLink>>>,
|
|
||||||
#[serde(rename = "link")]
|
|
||||||
pub link: Option<String>,
|
|
||||||
pub description: Option<String>,
|
|
||||||
#[serde(rename = "lastBuildDate")]
|
|
||||||
pub last_build_date: Option<String>,
|
|
||||||
pub language: Option<String>,
|
|
||||||
#[serde(rename = "sy:updatePeriod")]
|
|
||||||
pub update_period: Option<String>,
|
|
||||||
#[serde(rename = "sy:updateFrequency")]
|
|
||||||
pub update_frequency: Option<String>,
|
|
||||||
pub generator: Option<String>,
|
|
||||||
pub image: Option<Rc<RefCell<Image>>>,
|
|
||||||
#[serde(rename = "item", default)]
|
|
||||||
pub items: Vec<Rc<RefCell<Item>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
implement_from_lua!(Channel);
|
|
||||||
impl UserData for Channel {
|
|
||||||
fn add_fields<'lua, F: UserDataFields<'lua, Self>>(fields: &mut F) {
|
|
||||||
add_field_method_get_set!(fields, "title", title, Option<String>);
|
|
||||||
add_field_method_get_rc!(fields, "atom_link", atom_link);
|
|
||||||
add_field_method_set_rc_option!(fields, "atom_link", atom_link, AtomLink);
|
|
||||||
add_field_method_get_set!(fields, "link", link, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "description", description, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "last_build_date", last_build_date, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "language", language, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "update_period", update_period, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "update_frequency", update_frequency, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "generator", generator, Option<String>);
|
|
||||||
|
|
||||||
add_field_method_get_rc!(fields, "image", image);
|
|
||||||
add_field_method_set_rc_option!(fields, "image", image, Image);
|
|
||||||
add_field_method_get_rc!(fields, "articles", items);
|
|
||||||
fields.add_field_method_set("articles", |_, this, items: Vec<Item>| {
|
|
||||||
this.items = items
|
|
||||||
.into_iter()
|
|
||||||
.map(|i| Rc::new(RefCell::new(i)))
|
|
||||||
.collect();
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method("new", |_, _, ()| Ok(Channel::default()));
|
|
||||||
methods.add_method_mut("add_item", |_, this, item: Item| {
|
|
||||||
this.items.push(Rc::new(RefCell::new(item)));
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default, Debug, Deserialize, Serialize, Clone)]
|
|
||||||
pub struct AtomLink {
|
|
||||||
href: Option<String>,
|
|
||||||
rel: Option<String>,
|
|
||||||
#[serde(rename = "type")]
|
|
||||||
mime_type: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
implement_from_lua!(AtomLink);
|
|
||||||
|
|
||||||
impl UserData for AtomLink {
|
|
||||||
fn add_fields<'lua, F: UserDataFields<'lua, Self>>(fields: &mut F) {
|
|
||||||
add_field_method_get_set!(fields, "href", href, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "rel", rel, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "mime_type", mime_type, Option<String>);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method("new", |_, _, ()| Ok(AtomLink::default()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default, Debug, Deserialize, Serialize, Clone)]
|
|
||||||
pub struct Image {
|
|
||||||
url: Option<String>, // Option for missing fields
|
|
||||||
title: Option<String>,
|
|
||||||
link: Option<String>,
|
|
||||||
width: Option<u32>,
|
|
||||||
height: Option<u32>,
|
|
||||||
}
|
|
||||||
|
|
||||||
implement_from_lua!(Image);
|
|
||||||
impl UserData for Image {
|
|
||||||
fn add_fields<'lua, F: UserDataFields<'lua, Self>>(fields: &mut F) {
|
|
||||||
add_field_method_get_set!(fields, "url", url, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "title", title, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "link", link, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "width", width, Option<u32>);
|
|
||||||
add_field_method_get_set!(fields, "height", height, Option<u32>);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method("new", |_, _, ()| Ok(Image::default()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Lua object representing an RSS item as Article
|
|
||||||
#[derive(Default, Debug, Deserialize, Serialize, Clone)]
|
|
||||||
pub struct Item {
|
|
||||||
pub title: Option<String>, // Marked Option
|
|
||||||
pub link: Option<String>,
|
|
||||||
#[serde(rename = "dc:creator")]
|
|
||||||
pub creator: Option<String>,
|
|
||||||
#[serde(rename = "pubDate")]
|
|
||||||
pub pub_date: Option<String>,
|
|
||||||
#[serde(rename = "category", default)]
|
|
||||||
pub categories: Vec<String>,
|
|
||||||
pub guid: Option<Guid>,
|
|
||||||
pub description: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Item {
|
|
||||||
pub fn new(
|
|
||||||
guid: &str,
|
|
||||||
creator: &str,
|
|
||||||
categories: &str,
|
|
||||||
title: &str,
|
|
||||||
link: &str,
|
|
||||||
pub_date: &str,
|
|
||||||
description: &str,
|
|
||||||
) -> Item {
|
|
||||||
Item {
|
|
||||||
title: Some(title.to_string()),
|
|
||||||
link: Some(link.to_string()),
|
|
||||||
creator: Some(creator.to_string()),
|
|
||||||
pub_date: Some(pub_date.to_string()),
|
|
||||||
categories: categories.split(',').map(|s| s.to_string()).collect(),
|
|
||||||
guid: Some(Guid {
|
|
||||||
is_perma_link: Some(true),
|
|
||||||
value: Some(guid.to_string()),
|
|
||||||
}),
|
|
||||||
description: Some(description.to_string()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
implement_from_lua!(Item);
|
|
||||||
impl UserData for Item {
|
|
||||||
fn add_fields<'lua, F: UserDataFields<'lua, Self>>(fields: &mut F) {
|
|
||||||
add_field_method_get_set!(fields, "title", title, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "link", link, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "creator", creator, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "pub_date", pub_date, Option<String>);
|
|
||||||
add_field_method_get_set!(fields, "categories", categories, Vec<String>);
|
|
||||||
add_field_method_get_set!(fields, "guid", guid, Option<Guid>);
|
|
||||||
add_field_method_get_set!(fields, "description", description, Option<String>);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method("new", |_, _, ()| Ok(Item::default()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default, Debug, Deserialize, Serialize, Clone)]
|
|
||||||
pub struct Guid {
|
|
||||||
#[serde(rename = "isPermaLink")]
|
|
||||||
pub is_perma_link: Option<bool>,
|
|
||||||
#[serde(rename = "$value")]
|
|
||||||
pub value: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
implement_from_lua!(Guid);
|
|
||||||
impl UserData for Guid {
|
|
||||||
fn add_fields<'lua, F: UserDataFields<'lua, Self>>(fields: &mut F) {
|
|
||||||
add_field_method_get_set!(fields, "is_perma_link", is_perma_link, Option<bool>);
|
|
||||||
add_field_method_get_set!(fields, "value", value, Option<String>);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method("new", |_, _, ()| Ok(Guid::default()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,57 +0,0 @@
|
||||||
use anyhow::Result;
|
|
||||||
use chrono::prelude::*;
|
|
||||||
use chrono::Utc;
|
|
||||||
use log::debug;
|
|
||||||
use log::error;
|
|
||||||
use std::str::FromStr;
|
|
||||||
use std::thread;
|
|
||||||
|
|
||||||
use crate::scripting::ScriptingEngine;
|
|
||||||
|
|
||||||
pub struct Scheduler {
|
|
||||||
engine: ScriptingEngine,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Scheduler {
|
|
||||||
pub fn new(scripting: ScriptingEngine) -> Result<Self> {
|
|
||||||
Ok(Self { engine: scripting })
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_with_db(scripting: ScriptingEngine) -> Self {
|
|
||||||
Self { engine: scripting }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Example of a cron expression every minute: `0/1 * * * *`
|
|
||||||
pub fn schedule(&mut self, cron_expr: &str) -> Result<()> {
|
|
||||||
let schedule = cron::Schedule::from_str(cron_expr).unwrap();
|
|
||||||
while let Some(next) = schedule.upcoming(Utc).next() {
|
|
||||||
debug!("Scheduled task has started running at {}", Local::now());
|
|
||||||
self.engine.execute_all_routes()?;
|
|
||||||
debug!("Successfully executed all routes at {}", Local::now());
|
|
||||||
|
|
||||||
// Calculate the delta between now and the next scheduled run
|
|
||||||
let local: DateTime<Local> = DateTime::from(next);
|
|
||||||
let now = Utc::now();
|
|
||||||
|
|
||||||
// TODO: Check if scheduling 1 minute after now to prevent scheduling the same minute again
|
|
||||||
let until = next - now + chrono::Duration::minutes(1);
|
|
||||||
|
|
||||||
let sleep_time = match until.to_std() {
|
|
||||||
Ok(duration) => duration,
|
|
||||||
Err(_) => {
|
|
||||||
error!("Missed schedule, scheduling next run");
|
|
||||||
error!("A script took too long to run, consider increasing the interval or there may be a misbehaving script");
|
|
||||||
let next = schedule.upcoming(Utc).next().unwrap();
|
|
||||||
let until = next - now;
|
|
||||||
until.to_std().unwrap()
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
debug!("Next run UTC: {:?}", next);
|
|
||||||
debug!("Next run local: {:?}", local);
|
|
||||||
|
|
||||||
thread::sleep(sleep_time);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
342
src/scripting.rs
342
src/scripting.rs
|
|
@ -1,342 +0,0 @@
|
||||||
use crate::database::Database;
|
|
||||||
use crate::rss_parser::{Channel, Item, Rss};
|
|
||||||
use crate::REDIS;
|
|
||||||
use crate::{html_parser::HtmlParser, rss_parser::AtomLink};
|
|
||||||
use redis::Commands;
|
|
||||||
use log::{debug, error, info, trace, warn};
|
|
||||||
use mlua::{
|
|
||||||
chunk, ExternalResult, Function, Lua, MetaMethod, Result, Table, UserData, UserDataMethods
|
|
||||||
};
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
pub struct ScriptingEngine {
|
|
||||||
lua: Lua,
|
|
||||||
db: Database,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ScriptingEngine {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
let db = Database::new();
|
|
||||||
Self {
|
|
||||||
lua: Lua::new(),
|
|
||||||
db,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_with_db(db: Database) -> Self {
|
|
||||||
Self {
|
|
||||||
lua: Lua::new(),
|
|
||||||
db,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn load_all_scripts(&self) -> Result<()> {
|
|
||||||
// Look in the `scripts` directory for all Lua scripts
|
|
||||||
// and load them into the Lua environment
|
|
||||||
let scripts = std::fs::read_dir("scripts")?;
|
|
||||||
for script in scripts {
|
|
||||||
let script = script?;
|
|
||||||
if script.file_type()?.is_dir()
|
|
||||||
|| script.file_name().to_str().unwrap().starts_with(".")
|
|
||||||
|| !script.file_name().to_str().unwrap().ends_with(".lua")
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
let script_path = script.path();
|
|
||||||
let script_path = script_path.to_str().unwrap();
|
|
||||||
let script = std::fs::read_to_string(script_path)?;
|
|
||||||
self.load_script(&script)?;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn load_script(&self, script: &str) -> Result<()> {
|
|
||||||
match self.lua.load(script).exec() {
|
|
||||||
Ok(_) => Ok(()),
|
|
||||||
Err(err) => {
|
|
||||||
//TODO: Should this be an error and stop loading everything?
|
|
||||||
error!("Error loading script: {}", err);
|
|
||||||
error!("Not loading script:\n{}", script);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn load_script_with_return(&self, script: &str) -> Result<String> {
|
|
||||||
let result = self.lua.load(script).eval::<String>();
|
|
||||||
match result {
|
|
||||||
Ok(result) => Ok(result),
|
|
||||||
Err(err) => {
|
|
||||||
error!("Error loading script: {}", err);
|
|
||||||
Err(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn load_globals(&self) -> Result<()> {
|
|
||||||
macro_rules! add_constructor {
|
|
||||||
($name:literal, $constructor:ident) => {
|
|
||||||
let constructor = self
|
|
||||||
.lua
|
|
||||||
.create_function(|_, ()| Ok($constructor::default()))?;
|
|
||||||
self.lua.globals().set($name, constructor)?;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_constructor_new {
|
|
||||||
($name:literal, $constructor:ident) => {
|
|
||||||
let constructor = self.lua.create_function(|_, ()| Ok($constructor::new()))?;
|
|
||||||
self.lua.globals().set($name, constructor)?;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! add_global {
|
|
||||||
($name:literal, $value:expr) => {
|
|
||||||
self.lua.globals().set($name, $value)?;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
add_global!("log", Log);
|
|
||||||
//TODO: This is an async function... may be able to be used, but not on a tokio thread
|
|
||||||
//add_global!("get", self.lua.create_async_function(|_, url: String| async move {
|
|
||||||
// debug!("Making a GET request to {}", url);
|
|
||||||
// let response = reqwest::get(&url)
|
|
||||||
// .await
|
|
||||||
// .and_then(|response| response.error_for_status())
|
|
||||||
// .into_lua_err()
|
|
||||||
// .unwrap();
|
|
||||||
// debug!("Response: {:?}", response);
|
|
||||||
// Ok(response.text().await.unwrap())
|
|
||||||
//})?);
|
|
||||||
|
|
||||||
add_global!(
|
|
||||||
"get",
|
|
||||||
self.lua.create_function(|_, url: String| {
|
|
||||||
debug!("Making a GET request to {}", url);
|
|
||||||
let response = reqwest::blocking::get(&url)
|
|
||||||
.and_then(|response| response.error_for_status())
|
|
||||||
.into_lua_err()
|
|
||||||
.unwrap();
|
|
||||||
debug!("Response: {:?}", response);
|
|
||||||
Ok(response.text().unwrap())
|
|
||||||
})?
|
|
||||||
);
|
|
||||||
add_global!(
|
|
||||||
"sleep",
|
|
||||||
self.lua.create_function(|_, milliseconds: u64| {
|
|
||||||
debug!("Sleeping for {} milliseconds", milliseconds);
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(milliseconds));
|
|
||||||
Ok(())
|
|
||||||
})?
|
|
||||||
);
|
|
||||||
|
|
||||||
add_constructor!("HtmlParser", HtmlParser);
|
|
||||||
add_constructor!("Feed", Rss);
|
|
||||||
add_constructor!("Channel", Channel);
|
|
||||||
add_constructor!("AtomLink", AtomLink);
|
|
||||||
add_constructor!("Article", Item);
|
|
||||||
|
|
||||||
let database = Database::new();
|
|
||||||
self.lua.globals().set("db", database)?;
|
|
||||||
|
|
||||||
self.lua.globals().set("routes", self.lua.create_table()?)?;
|
|
||||||
let add_route = chunk! {
|
|
||||||
function add_route(table, route)
|
|
||||||
log:info("Adding route " .. route .. " to table " .. table)
|
|
||||||
if string.sub(route, 1, 1) == "/" then
|
|
||||||
route = string.sub(route, 2)
|
|
||||||
end
|
|
||||||
routes[route] = table
|
|
||||||
end
|
|
||||||
};
|
|
||||||
self.lua.load(add_route).exec()?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Execute all of the routes that are registered
|
|
||||||
pub fn execute_all_routes(&mut self) -> Result<()> {
|
|
||||||
let routes = self.get_routes().unwrap();
|
|
||||||
debug!("Executing all lua script routes");
|
|
||||||
debug!("Routes: {:?}", routes);
|
|
||||||
|
|
||||||
for (name, route) in routes {
|
|
||||||
debug!("Executing route: {}", name);
|
|
||||||
let ret = match self.execute_route_function(&route, &name) {
|
|
||||||
Ok(ret) => ret,
|
|
||||||
Err(_) => {
|
|
||||||
error!("Removing misbehaving route: {}", name);
|
|
||||||
self.remove_route(&name)?;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let result = ret.0;
|
|
||||||
let feed = ret.1;
|
|
||||||
|
|
||||||
trace!("Route {} executed with result: {}", name, result);
|
|
||||||
let _: () = match REDIS.lock().unwrap().set(name, result) {
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(err) => {
|
|
||||||
error!("Error setting key in Redis: {}", err);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
//TODO: Put on another thread?
|
|
||||||
debug!("Inserting articles into database");
|
|
||||||
let channel = feed.channel;
|
|
||||||
let articles = &channel.borrow().items;
|
|
||||||
for article in articles {
|
|
||||||
let article = article.borrow();
|
|
||||||
match self.db.insert_article(&article) {
|
|
||||||
Ok(_) => (),
|
|
||||||
Err(_) => continue,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
debug!("Finished inserting articles into database");
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Execute the route function from the object route passed in
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `route_name` - The name of the route to execute
|
|
||||||
pub fn execute_route_function(
|
|
||||||
&self,
|
|
||||||
obj_name: &str,
|
|
||||||
route_name: &str,
|
|
||||||
) -> Result<(String, Rss)> {
|
|
||||||
let route_obj: Table = match self.lua.globals().get(obj_name) {
|
|
||||||
Ok(route) => route,
|
|
||||||
Err(_) => {
|
|
||||||
error!("Object {} does not exist", obj_name);
|
|
||||||
return Ok(("".to_string(), Rss::default()));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let route_func: Function = match route_obj.get("route") {
|
|
||||||
Ok(route) => route,
|
|
||||||
Err(_) => {
|
|
||||||
error!("Route {} does not have a route function", obj_name);
|
|
||||||
error!("Removing route {}", obj_name);
|
|
||||||
self.remove_route(route_name)?;
|
|
||||||
return Ok(("".to_string(), Rss::default()));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let result = match route_func.call::<_, (String, Rss)>(()) {
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(e) => {
|
|
||||||
// Missing the right return values drops down here
|
|
||||||
error!("Error executing route function for route {}", obj_name);
|
|
||||||
error!("Lua error: {}", e);
|
|
||||||
return Err(mlua::Error::external("Error executing route function"));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
Ok(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_routes(&self) -> Result<HashMap<String, String>> {
|
|
||||||
let routes = self
|
|
||||||
.lua
|
|
||||||
.globals()
|
|
||||||
.get::<_, HashMap<String, String>>("routes")?;
|
|
||||||
Ok(routes)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn remove_route(&self, route: &str) -> Result<()> {
|
|
||||||
let remove_route = chunk! {
|
|
||||||
routes[$route] = nil
|
|
||||||
};
|
|
||||||
self.lua.load(remove_route).exec()?;
|
|
||||||
debug!("Route {} removed", route);
|
|
||||||
debug!("Routes: {:?}", self.get_routes()?);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
pub struct Log;
|
|
||||||
impl UserData for Log {
|
|
||||||
fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
|
|
||||||
methods.add_method("info", |_, _, msg: String| {
|
|
||||||
info!("{}", msg);
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
|
|
||||||
methods.add_method("debug", |_, _, msg: String| {
|
|
||||||
debug!("{}", msg);
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
|
|
||||||
methods.add_method("warn", |_, _, msg: String| {
|
|
||||||
warn!("{}", msg);
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
|
|
||||||
methods.add_method("error", |_, _, msg: String| {
|
|
||||||
error!("{}", msg);
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
|
|
||||||
methods.add_meta_function(MetaMethod::Call, |_, ()| Ok(()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use mlua::chunk;
|
|
||||||
use rusqlite::Connection;
|
|
||||||
|
|
||||||
use super::*;
|
|
||||||
use crate::rss_parser::Guid;
|
|
||||||
use crate::rss_parser::Item as Article;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_get_article_by_guid_from_lua() -> Result<()> {
|
|
||||||
let conn = match Connection::open_in_memory() {
|
|
||||||
Ok(conn) => conn,
|
|
||||||
Err(err) => {
|
|
||||||
return Err(mlua::Error::external(err));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let db = Database::new_with_conn(conn);
|
|
||||||
let article = Article {
|
|
||||||
title: Some("Test".to_string()),
|
|
||||||
guid: Some(Guid {
|
|
||||||
value: Some("https://techcrunch.com/?p=2877786".to_string()),
|
|
||||||
is_perma_link: Some(false),
|
|
||||||
}),
|
|
||||||
creator: None,
|
|
||||||
categories: vec![],
|
|
||||||
link: Some("https://example.com".to_string()),
|
|
||||||
pub_date: Some("2021-01-01T00:00:00Z".to_string()),
|
|
||||||
description: Some("Test".to_string()),
|
|
||||||
};
|
|
||||||
let ret = db.insert_article(&article);
|
|
||||||
assert!(ret.is_ok());
|
|
||||||
|
|
||||||
let script = chunk! {
|
|
||||||
local db = $db;
|
|
||||||
local article = db:get_article_by_guid("https://techcrunch.com/?p=2877786");
|
|
||||||
return article;
|
|
||||||
};
|
|
||||||
let engine = ScriptingEngine::new();
|
|
||||||
engine.load_globals()?;
|
|
||||||
|
|
||||||
let ret = engine.lua.load(script).eval::<Article>()?;
|
|
||||||
assert_eq!(ret.title.unwrap(), "Test");
|
|
||||||
assert_eq!(
|
|
||||||
ret.guid.unwrap().value.unwrap(),
|
|
||||||
"https://techcrunch.com/?p=2877786"
|
|
||||||
);
|
|
||||||
assert_eq!(ret.link.unwrap(), "https://example.com");
|
|
||||||
assert_eq!(ret.pub_date.unwrap(), "2021-01-01T00:00:00Z");
|
|
||||||
assert_eq!(ret.description.unwrap(), "Test");
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue