get_unique_updates Initial commit

Script which can take the markdown output of worklog.py and
 output one entry per package using the highest version

common/.gitignore

@@ -1 +1,3 @@
-*.pyc
+*.pyc
+*.vstags
+*.bak

common/perl/README

@@ -0,0 +1,25 @@
+# README for perl scripts
+
+## Documentation and help output
+
+For each script, there should be documentation written in pod and help output.
+
+Docs are read with perldoc. For example:
+
+```bash
+perldoc get_unique_updates/get_unique_updates.pl
+```
+
+Each script *should* have help output. For example:
+
+```bash
+get_unique_updates.pl --help
+```
+
+## Running tests
+
+Tests are run with `prove`. To run all tests for a script, run prove on all the test files in the test directory:
+
+```bash
+prove get_unique_updates/t/*.t
+```

common/perl/get_unique_updates/get_unique_updates.pl

@@ -0,0 +1,89 @@
+#!/usr/bin/env perl
+
+# ABSTRACT: Find and return all unique package names in input file with highest version string
+# PODNAME: get_unique_updates.pl
+
+use Modern::Perl;
+
+use FindBin;
+use lib "$FindBin::RealBin/lib";
+use GetUniqueLines;
+
+use Getopt::Long;
+use Pod::Usage;
+
+my $successful_parse = GetOptions(
+   'help|?'        => \my $help,
+   man             => \my $man,
+   'input-file=s'  => \my $input_file,
+   'output-dir=s'  => \( my $output_dir = "$FindBin::RealBin" ),
+   'output-file=s' => \my $output_file
+  )
+  or pod2usage(2);
+
+if ( !$successful_parse || ( @ARGV > 2 ) ) {
+   die qq{Usage: $0 [ --input-file input_filename --output-dir output_directory]\n};
+}
+
+if ( !defined $input_file ) {
+   die qq{   An input file is required\n
+      Usage: $0 [ --input-file input_filename --output-dir output_directory --output-file output_filename]\n};
+}
+
+pod2usage(1)                              if $help;
+pod2usage( -exitval => 0, -verbose => 2 ) if $man;
+
+unless ( -e -f -r $input_file ) {
+   die("Unable to read '$input_file': $!\n");
+}
+
+unless ( GetUniqueLines::check_file_format($input_file) ) {
+   die("File '$input_file' is not in the expected markdown format\n");
+}
+
+my %packages = GetUniqueLines::get_unique_lines($input_file);
+my $pkg_href = \%packages;
+
+# Write to STDOUT, unless output file provided
+if ($output_file) {
+   my $file_path = $output_dir . '/' . $output_file;
+   GetUniqueLines::write_to_file( $file_path, $pkg_href );
+} else {
+   GetUniqueLines::write_to_stdout($pkg_href);
+}
+
+__END__
+=pod
+
+=encoding UTF-8
+
+=head1 NAME
+
+get_unique_updates.pl
+
+=head1 SYNOPSIS
+
+get_unique_updates.pl --input-file ~/temp/ --output-dir ~/temp
+
+ Options:
+   -help                Brief help message
+   -man                 Full documentation
+   --input-file         The file to parse, generated by worklog.py (required)
+   --output-dir         Directory to store output file (defaults to the directory this script is in, only used if --output-file is provided)
+   --output-file         The name of the file to save the output to. If this is not provided, the list of packages will be printed to STDOUT.
+
+=head1 DESCRIPTION
+
+Solus uses worklog.py to generate a list of packages that were updated in a given timeframe. The file it generates includes all updates to every package.
+
+This script takes that file as input and returns a list to STDOUT or a file that has only one line per package. That line will be the latest version built for the package.
+
+You must use the markdown output type with worklog.py for use with this script. e.g.
+
+    ./worklog.py --format md builds 2024-02-04 now >| ~/temp/solus_builds_2024-02-04.md
+
+Examples:
+
+    get_unique_updates.pl --input-file ~/temp/solus_builds_2024-02-04.md
+
+    get_unique_updates.pl --input-file ~/temp/solus_builds_2024-02-04.md --output-dir ~/temp --output-file solus_unique_builds_2024-02-04.md

common/perl/get_unique_updates/lib/GetUniqueLines.pm

@@ -0,0 +1,99 @@
+package GetUniqueLines;
+
+use Modern::Perl;
+use List::Util;
+
+use Data::Dumper;
+
+use Pod::Usage;
+
+sub check_file_format {
+   my $file = shift;
+   my $is_markdown;
+
+   # Make sure file is in the right format
+   # This is a very simple check to avoid needing to have more perl modules installed
+
+   open( my $fh, "<", $file )
+     or die "Can't open '$file': $!";
+   my @rows = <$fh>;
+   chomp @rows;
+
+   # Line one should be a number of builds
+   unless ( $rows[0] =~ /\d+\sbuilds/i ) {
+      die( "First line of the filed does not appear to show the number of builds. Got:\n'" . $rows[0] . "'\n" );
+   }
+
+   # Line two should be an unordered list item which is a package name and link
+   unless ( $rows[1] =~ /^-\s\[[^\s]+\s\d+[^\s]+\]\(https:\/\/[^\s]+\)/ ) {
+      die( "Second line of the file is not in expected markdown format. Got:\n'" . $rows[1] . "'\n" );
+   }
+   $is_markdown = 1;
+
+   close($fh) or die $!;
+
+   return $is_markdown;
+}
+
+sub get_unique_lines {
+
+   # Take input file
+   my $input_file = shift;
+   my %packages;
+
+   open( my $fh, "<", $input_file )
+     or die "Can't open file '$input_file': $!";
+   my @lines = <$fh>;
+   chomp @lines;
+
+   # iterate through lines and parse out package names
+   # Create a hash of packages, package names are the keys
+   # Each update of each key will write in the line with the highest version
+   # Skip first line since that is the package count
+   for my $i ( 1 .. $#lines ) {
+      my $line = $lines[$i];
+
+      # Get package name
+      if ( $line =~ /^-\s\[([^\s]+)\s/ ) {
+         my $pkg_name = $1;
+
+         #  say "Got package name $1";
+         $packages{$pkg_name} = $line;
+      } else {
+         next;
+      }
+   }
+
+   # return the hash of unique package lines
+   return %packages;
+}
+
+sub write_to_stdout {
+   my $package_href = shift;
+   my %package_list = %{$package_href};
+
+   foreach my $name ( sort( keys %package_list ) ) {
+      print $package_list{$name} . "\n";
+   }
+
+   return;
+}
+
+sub write_to_file {
+   my ( $file_path, $package_href ) = @_;
+   my %package_list = %{$package_href};
+
+   open( my $fh, ">", $file_path )
+     or die "Can't open '$file_path': $!";
+
+   foreach my $name ( sort( keys %package_list ) ) {
+      print $fh $package_list{$name} . "\n";
+   }
+
+   close($fh) or die $!;
+
+   return;
+
+}
+
+1;

common/perl/get_unique_updates/t/data/markdown_text.md

@@ -0,0 +1,20 @@
+123 builds:
+- [0ad 0.0.26-53](https://github.com/getsolus/packages/commit/bfef63e3245209ab3b32ddb6b11d93681a904e12)
+- [abseil-cpp 20230802.1-5](https://github.com/getsolus/packages/commit/354b2c458704bb639e6437b44dd668dace98c63b)
+- [abseil-cpp 20230802.1-6](https://github.com/getsolus/packages/commit/91739a1a49c48e68279370ba5400059a7dcef840)
+- [aegisub 3.3.3-39](https://github.com/getsolus/packages/commit/d86decc52fcd44ea02a7f999ad5870bab781d977)
+- [alsa-lib 1.2.11-37](https://github.com/getsolus/packages/commit/83989aad2c1d565c92ba6c2a9b4b459a869495ab)
+- [alsa-tools 1.2.11-14](https://github.com/getsolus/packages/commit/0f355093daca07dbed0adc630d154db8868ad4de)
+- [alsa-utils 1.2.11-27](https://github.com/getsolus/packages/commit/33cef9e6346e8cd7ce96731a9cf3842a36a5287f)
+- [amdgpu_top 0.7.0-10](https://github.com/getsolus/packages/commit/bdcfc6f42d356ce5c08812a535e35b3054c71b4d)
+- [anki 23.12.1-33](https://github.com/getsolus/packages/commit/113bae3c5a250823065b5f09f0da9821164579d5)
+- [appstream-data 32-35](https://github.com/getsolus/packages/commit/d8c7882c390d10e82748d9ddd04b306bb3415c0d)
+- [apr-util 1.6.3-18](https://github.com/getsolus/packages/commit/ef1b3da340dd02c5df6665b1a87d46102cc4a30b)
+- [attica 5.115.0-74](https://github.com/getsolus/packages/commit/40d198c4c875d288646358fdf202f4ecddeda389)
+- [autofs 5.1.9-12](https://github.com/getsolus/packages/commit/b6edcf792be13222ac8a8a3e13545f0fad1122ba)
+- [baloo 5.115.0-75](https://github.com/getsolus/packages/commit/630558119e3dec18ce36aeed0f6d51146e855b1d)
+- [broadcom-sta 6.30.223.271-384](https://github.com/getsolus/packages/commit/57ba69fbadfde4170037f1af1fca46213154c01e)
+- [broadcom-sta 6.30.223.271-385](https://github.com/getsolus/packages/commit/d633bea9ffa45f50d0aff82d2c943ceea96a15f6)
+- [darktable 4.6.0-75](https://github.com/getsolus/packages/commit/acecb5d8c41d18c3a2b81a7436e6ba5525c32ae7)
+- [darktable 4.6.0-76](https://github.com/getsolus/packages/commit/1f8be1a336c6c5ed6966ae32846b7a6b31c13286)
+- [darktable 4.6.0-77](https://github.com/getsolus/packages/commit/3e12b4295851f3325da4d4f43ff1cd4e093d65eb)

common/perl/get_unique_updates/t/data/random_text

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

common/perl/get_unique_updates/t/get_unique_updates.t

@@ -0,0 +1,68 @@
+use Test2::V0;
+use Test2::Tools::Exception qw/dies lives/;
+use Test2::Plugin::DieOnFail;
+use Test2::Plugin::ExitSummary;
+use Test::File;
+
+use FindBin qw($Bin);
+use lib "$Bin/../lib";
+
+my $text_file   = "$Bin/../t/data/random_text";
+my $md_file     = "$Bin/../t/data/markdown_text.md";
+my $output_dir  = "$Bin/../t/data";
+my $output_file = "markdown_output.md";
+my $file_path   = $output_dir . '/' . $output_file;
+
+my $expected_broadcom_ver  = 'broadcom-sta 6.30.223.271-385';
+my $expected_darktable_ver = 'darktable 4.6.0-77';
+
+BEGIN {
+   plan(6);
+   use GetUniqueLines;
+   pass('Module loads correctly.');
+}
+
+ok( GetUniqueLines::check_file_format($md_file), "Markdown file passes file check" );
+
+like( dies { GetUniqueLines::check_file_format($text_file) }, qr/First line/, "Text file fails file check" );
+
+# Make sure we get highest package versions from the result
+my %packages = GetUniqueLines::get_unique_lines($md_file);
+my $pkg_href = \%packages;
+
+# Capture output of write_to_stdout
+my $stdout_result;
+open( my $outputFH, '>', \$stdout_result ) or die;
+my $oldFH = select $outputFH;
+GetUniqueLines::write_to_stdout($pkg_href);
+select $oldFH;
+close $outputFH;
+
+my ( $broadcom_result, $darktable_result );
+
+if ( $stdout_result =~ /\[(broadcom-sta \S+)\]/g ) {
+   $broadcom_result = $1;
+   like( "$broadcom_result", qr/$expected_broadcom_ver/, "Entry for package with 2 versions has highest value" );
+} else {
+   fail('Broadcom entry found in result');
+}
+
+if ( $stdout_result =~ /\[(darktable \S+)\]/g ) {
+   $darktable_result = $1;
+   like( "$darktable_result", qr/$expected_darktable_ver/, "Entry for package with 3 versions has highest version" );
+} else {
+   fail('Darktable entry found in result');
+}
+
+# Ensure module can write a file
+
+GetUniqueLines::write_to_file( $file_path, $pkg_href );
+
+print "test looking for $file_path\n";
+# file_exists_ok( $file_path, 'Output file is written');
+file_line_count_is( $file_path, 15, 'Output file is written and has expected number of packages');
+
+END {
+    # Remove test file
+    unlink($file_path) or die "Can't delete $file_path: $!\n";
+}