mirror of
https://github.com/Piwigo/Piwigo.git
synced 2025-04-30 05:09:57 +03:00
merge r3192 from branch 2.0 to trunk
bug 941 fixed: to be able to upload heavy photo, chunk the files, send parts one by one, and then pwg.images.add merge chunks together. Now big uploads works and you can even have a fine progress bar on client side. git-svn-id: http://piwigo.org/svn/trunk@3193 68402e56-0260-453c-a942-63ccdbb3a9ee
This commit is contained in:
plegall
parent
bd005369b0
commit
f94ff8b8e1
3 changed files with 207 additions and 45 deletions
|
|
@ -879,6 +879,98 @@ UPDATE '.IMAGES_TABLE.'
|
|||
return $affected_rows;
|
||||
}
|
||||
|
||||
function ws_images_add_chunk($params, &$service)
|
||||
{
|
||||
// data
|
||||
// original_sum
|
||||
// type {thumb, file, high}
|
||||
// position
|
||||
|
||||
if (!is_admin() || is_adviser() )
|
||||
{
|
||||
return new PwgError(401, 'Access denied');
|
||||
}
|
||||
|
||||
$upload_dir = PHPWG_ROOT_PATH.'upload/buffer';
|
||||
|
||||
// create the upload directory tree if not exists
|
||||
if (!is_dir($upload_dir)) {
|
||||
umask(0000);
|
||||
$recursive = true;
|
||||
if (!@mkdir($upload_dir, 0777, $recursive))
|
||||
{
|
||||
return new PwgError(500, 'error during buffer directory creation');
|
||||
}
|
||||
}
|
||||
|
||||
if (!is_writable($upload_dir))
|
||||
{
|
||||
// last chance to make the directory writable
|
||||
@chmod($upload_dir, 0777);
|
||||
|
||||
if (!is_writable($upload_dir))
|
||||
{
|
||||
return new PwgError(500, 'buffer directory has no write access');
|
||||
}
|
||||
}
|
||||
|
||||
secure_directory($upload_dir);
|
||||
|
||||
$filename = sprintf(
|
||||
'%s-%s-%05u.block',
|
||||
$params['original_sum'],
|
||||
$params['type'],
|
||||
$params['position']
|
||||
);
|
||||
|
||||
$bytes_written = file_put_contents(
|
||||
$upload_dir.'/'.$filename,
|
||||
$params['data']
|
||||
);
|
||||
|
||||
if (false === $bytes_written) {
|
||||
return new PwgError(
|
||||
500,
|
||||
'an error has occured while writting chunk '.$params['position'].' for '.$params['type']
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function merge_chunks($output_filepath, $original_sum, $type)
|
||||
{
|
||||
ws_logfile('[merge_chunks] input parameter $output_filepath : '.$output_filepath);
|
||||
|
||||
$upload_dir = PHPWG_ROOT_PATH.'upload/buffer';
|
||||
$pattern = '/'.$original_sum.'-'.$type.'/';
|
||||
$chunks = array();
|
||||
|
||||
if ($handle = opendir($upload_dir))
|
||||
{
|
||||
while (false !== ($file = readdir($handle)))
|
||||
{
|
||||
if (preg_match($pattern, $file))
|
||||
{
|
||||
ws_logfile($file);
|
||||
array_push($chunks, $upload_dir.'/'.$file);
|
||||
}
|
||||
}
|
||||
closedir($handle);
|
||||
}
|
||||
|
||||
sort($chunks);
|
||||
|
||||
$string = null;
|
||||
foreach ($chunks as $chunk) {
|
||||
$string.= file_get_contents($chunk);
|
||||
unlink($chunk);
|
||||
}
|
||||
if (!file_put_contents($output_filepath, base64_decode($string)))
|
||||
{
|
||||
return new PwgError(500, 'error while merging chunks for '.$output_filepath);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function ws_images_add($params, &$service)
|
||||
{
|
||||
global $conf;
|
||||
|
|
@ -956,13 +1048,8 @@ SELECT
|
|||
$filename_wo_ext = $date_string.'-'.$random_string;
|
||||
$file_path = $upload_dir.'/'.$filename_wo_ext.'.jpg';
|
||||
|
||||
// dump the photo file
|
||||
$fh_file = fopen($file_path, 'w');
|
||||
if (!fwrite($fh_file, base64_decode($params['file_content'])))
|
||||
{
|
||||
return new PwgError(500, 'error while writing file');
|
||||
}
|
||||
fclose($fh_file);
|
||||
// merge the photo file
|
||||
merge_chunks($file_path, $params['original_sum'], 'file');
|
||||
chmod($file_path, 0644);
|
||||
|
||||
// check dumped file md5sum against expected md5sum
|
||||
|
|
@ -1005,13 +1092,8 @@ SELECT
|
|||
'jpg'
|
||||
);
|
||||
|
||||
// dump the thumbnail
|
||||
$fh_thumbnail = fopen($thumbnail_path, 'w');
|
||||
if (!fwrite($fh_thumbnail, base64_decode($params['thumbnail_content'])))
|
||||
{
|
||||
return new PwgError(500, 'error while writing thumbnail');
|
||||
}
|
||||
fclose($fh_thumbnail);
|
||||
// merge the thumbnail
|
||||
merge_chunks($thumbnail_path, $params['original_sum'], 'thumb');
|
||||
chmod($thumbnail_path, 0644);
|
||||
|
||||
// check dumped thumbnail md5
|
||||
|
|
@ -1021,7 +1103,7 @@ SELECT
|
|||
}
|
||||
|
||||
// high resolution
|
||||
if (isset($params['high_content']))
|
||||
if (isset($params['high_sum']))
|
||||
{
|
||||
// high resolution directory is a subdirectory of the photo file, hard
|
||||
// coded "pwg_high"
|
||||
|
|
@ -1055,13 +1137,8 @@ SELECT
|
|||
'jpg'
|
||||
);
|
||||
|
||||
// dump the high resolution file
|
||||
$fh_high = fopen($high_path, 'w');
|
||||
if (!fwrite($fh_high, base64_decode($params['high_content'])))
|
||||
{
|
||||
return new PwgError(500, 'error while writing high');
|
||||
}
|
||||
fclose($fh_high);
|
||||
// merge the high resolution file
|
||||
merge_chunks($high_path, $params['original_sum'], 'high');
|
||||
chmod($high_path, 0644);
|
||||
|
||||
// check dumped thumbnail md5
|
||||
|
|
@ -1104,7 +1181,7 @@ SELECT
|
|||
}
|
||||
}
|
||||
|
||||
if (isset($params['high_content']))
|
||||
if (isset($params['high_sum']))
|
||||
{
|
||||
$insert['has_high'] = 'true';
|
||||
$insert['high_filesize'] = $high_filesize;
|
||||
|
|
@ -1646,4 +1723,15 @@ SELECT
|
|||
update_category($cat_ids);
|
||||
}
|
||||
}
|
||||
|
||||
function ws_logfile($string)
|
||||
{
|
||||
return true;
|
||||
|
||||
file_put_contents(
|
||||
'/tmp/piwigo_ws.log',
|
||||
'['.date('c').'] '.$string."\n",
|
||||
FILE_APPEND
|
||||
);
|
||||
}
|
||||
?>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,17 @@
|
|||
#!/usr/bin/perl
|
||||
|
||||
####
|
||||
# Usage examples
|
||||
#
|
||||
# time perl piwigo_remote.pl \
|
||||
# --action=pwg.images.add \
|
||||
# --file=erwann_rocher-web.jpg \
|
||||
# --thumb=erwann_rocher-thumb.jpg \
|
||||
# --high=erwann_rocher-high.jpg \
|
||||
# --original=erwann_rocher-high.jpg \
|
||||
# --define categories=9 \
|
||||
# --chunk_size=200_000
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
|
|
@ -7,22 +19,24 @@ use JSON;
|
|||
use LWP::UserAgent;
|
||||
use Getopt::Long;
|
||||
use Encode qw/is_utf8 decode/;
|
||||
use POSIX qw(ceil floor);
|
||||
|
||||
my %opt = ();
|
||||
GetOptions(
|
||||
\%opt,
|
||||
qw/action=s file=s thumbnail=s high=s original=s categories=s define=s%/
|
||||
qw/action=s file=s thumbnail=s high=s original=s categories=s chunk_size=i define=s%/
|
||||
);
|
||||
|
||||
our $ua = LWP::UserAgent->new;
|
||||
$ua->cookie_jar({});
|
||||
|
||||
my %conf;
|
||||
$conf{base_url} = 'http://localhost/~pierrick/piwigo/trunk';
|
||||
$conf{base_url} = 'http://localhost/piwigo/2.0';
|
||||
$conf{response_format} = 'json';
|
||||
$conf{username} = 'pierrick';
|
||||
$conf{password} = 'z0rglub';
|
||||
$conf{username} = 'plg';
|
||||
$conf{password} = 'plg';
|
||||
$conf{limit} = 10;
|
||||
$conf{chunk_size} = defined $opt{chunk_size} ? $opt{chunk_size} : 500_000;
|
||||
|
||||
my $result = undef;
|
||||
my $query = undef;
|
||||
|
|
@ -48,26 +62,32 @@ if ($opt{action} eq 'pwg.images.add') {
|
|||
use Digest::MD5::File qw/file_md5_hex/;
|
||||
use File::Slurp;
|
||||
|
||||
$form = {};
|
||||
$form->{method} = 'pwg.images.add';
|
||||
|
||||
my $original_sum = file_md5_hex($opt{original});
|
||||
$form->{original_sum} = $original_sum;
|
||||
|
||||
my $file_content = encode_base64(read_file($opt{file}));
|
||||
my $file_sum = file_md5_hex($opt{file});
|
||||
|
||||
my $thumbnail_content = encode_base64(read_file($opt{thumbnail}));
|
||||
my $thumbnail_sum = file_md5_hex($opt{thumbnail});
|
||||
|
||||
$form = {
|
||||
method => 'pwg.images.add',
|
||||
send_chunks(
|
||||
filepath => $opt{file},
|
||||
type => 'file',
|
||||
original_sum => $original_sum,
|
||||
file_sum => $file_sum,
|
||||
file_content => $file_content,
|
||||
thumbnail_sum => $thumbnail_sum,
|
||||
thumbnail_content => $thumbnail_content,
|
||||
categories => $opt{categories},
|
||||
};
|
||||
);
|
||||
$form->{file_sum} = file_md5_hex($opt{file});
|
||||
|
||||
send_chunks(
|
||||
filepath => $opt{thumbnail},
|
||||
type => 'thumb',
|
||||
original_sum => $original_sum,
|
||||
);
|
||||
$form->{thumbnail_sum} = file_md5_hex($opt{thumbnail});
|
||||
|
||||
if (defined $opt{high}) {
|
||||
$form->{high_content} = encode_base64(read_file($opt{high}));
|
||||
send_chunks(
|
||||
filepath => $opt{high},
|
||||
type => 'high',
|
||||
original_sum => $original_sum,
|
||||
);
|
||||
$form->{high_sum} = file_md5_hex($opt{high});
|
||||
}
|
||||
|
||||
|
|
@ -231,3 +251,47 @@ sub pwg_ws_get_query {
|
|||
|
||||
return $query;
|
||||
}
|
||||
|
||||
sub send_chunks {
|
||||
my %params = @_;
|
||||
|
||||
my $content = encode_base64(read_file($params{filepath}));
|
||||
my $content_length = length($content);
|
||||
my $nb_chunks = ceil($content_length / $conf{chunk_size});
|
||||
|
||||
my $chunk_pos = 0;
|
||||
my $chunk_id = 1;
|
||||
while ($chunk_pos < $content_length) {
|
||||
my $chunk = substr(
|
||||
$content,
|
||||
$chunk_pos,
|
||||
$conf{chunk_size}
|
||||
);
|
||||
$chunk_pos += $conf{chunk_size};
|
||||
|
||||
my $response = $ua->post(
|
||||
$conf{base_url}.'/ws.php?format=json',
|
||||
{
|
||||
method => 'pwg.images.addChunk',
|
||||
data => $chunk,
|
||||
original_sum => $params{original_sum},
|
||||
position => $chunk_id,
|
||||
type => $params{type},
|
||||
}
|
||||
);
|
||||
|
||||
printf(
|
||||
'chunk %05u of %05u for %s "%s"'."\n",
|
||||
$chunk_id,
|
||||
$nb_chunks,
|
||||
$params{type},
|
||||
$params{filepath}
|
||||
);
|
||||
if ($response->code != 200) {
|
||||
printf("response code : %u\n", $response->code);
|
||||
printf("response message : %s\n", $response->message);
|
||||
}
|
||||
|
||||
$chunk_id++;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
16
ws.php
16
ws.php
|
|
@ -172,15 +172,25 @@ function ws_addDefaultMethods( $arr )
|
|||
'Returns elements for the corresponding tags. Note that tag_id, tag_url_name, tag_name an be arrays. Fill at least one of them. '
|
||||
);
|
||||
|
||||
$service->addMethod(
|
||||
'pwg.images.addChunk',
|
||||
'ws_images_add_chunk',
|
||||
array(
|
||||
'data' => array(),
|
||||
'original_sum' => array(),
|
||||
'type' => array(),
|
||||
'position' => array(),
|
||||
),
|
||||
'POST method only. For admin only.'
|
||||
);
|
||||
|
||||
|
||||
$service->addMethod(
|
||||
'pwg.images.add',
|
||||
'ws_images_add',
|
||||
array(
|
||||
'file_content' => array(),
|
||||
'file_sum' => array(),
|
||||
'thumbnail_content' => array(),
|
||||
'thumbnail_sum' => array(),
|
||||
'high_content' => array('default' => null),
|
||||
'high_sum' => array('default' => null),
|
||||
'original_sum' => array(),
|
||||
'name' => array('default' => null),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue