Skip to content

Commit dfcc2e6

Browse files
Nyholmjderusse
andauthoredOct 5, 2020
Adding support for AsyncAws (#639)
* Adding support for AsyncAws * Added PHPSpec test * cs * Update doc/adapters/async-aws-s3.md Co-authored-by: Jérémy Derussé <[email protected]> * Fixes * cs * cs * Minors * cs * fixes * minor * fix * Update make files Co-authored-by: Jérémy Derussé <[email protected]>
1 parent a786092 commit dfcc2e6

File tree

9 files changed

+568
-3
lines changed

9 files changed

+568
-3
lines changed
 

‎.github/workflows/ci.yml

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ jobs:
4242
max-parallel: 10
4343
matrix:
4444
run:
45+
- { php: '7.3', packages: 'async-aws/simple-s3:^0.1.1', phpspec: 'spec/Gaufrette/Adapter/AsyncAwsS3Spec.php' }
4546
- { php: '7.3', packages: 'aws/aws-sdk-php:^2.4.12', phpspec: 'spec/Gaufrette/Adapter/AwsS3Spec.php' }
4647
- { php: '7.4', packages: 'aws/aws-sdk-php:^3.158', phpspec: 'spec/Gaufrette/Adapter/AwsS3Spec.php' }
4748
- { php: '7.4', packages: 'rackspace/php-opencloud:^1.9.2', phpspec: 'spec/Gaufrette/Adapter/OpenCloudSpec.php' }

‎.travis.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ before_install:
3939
- docker exec -it ftpd_server sh -c "(echo ${FTP_PASSWORD}; echo ${FTP_PASSWORD}) | pure-pw useradd ${FTP_USER} -f /etc/pure-ftpd/passwd/pureftpd.passwd -m -u ftpuser -d /home/ftpusers/${FTP_USER}"
4040

4141
install:
42-
- make require-all
42+
- make require-all-legacy
4343
- composer update --prefer-dist --no-progress --no-suggest --ansi
4444

4545
script:

‎Makefile

+6-1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ php-cs-fix:
3333
docker/run-task php${PHP_VERSION} vendor/bin/php-cs-fixer fix
3434

3535
remove-phpspec:
36+
rm spec/Gaufrette/Adapter/AsyncAwsS3Spec.php
3637
rm spec/Gaufrette/Adapter/AwsS3Spec.php
3738
rm spec/Gaufrette/Adapter/OpenCloudSpec.php
3839
rm spec/Gaufrette/Adapter/GoogleCloudStorageSpec.php
@@ -42,7 +43,7 @@ remove-phpspec:
4243
rm spec/Gaufrette/Adapter/GridFSSpec.php
4344
rm spec/Gaufrette/Adapter/PhpseclibSftpSpec.php
4445

45-
require-all:
46+
require-all-legacy:
4647
composer require --no-update \
4748
aws/aws-sdk-php:^3.158 \
4849
rackspace/php-opencloud:^1.9.2 \
@@ -54,3 +55,7 @@ require-all:
5455
mongodb/mongodb:^1.1 \
5556
symfony/event-dispatcher:^4.4
5657

58+
59+
require-all: require-all-legacy
60+
composer require --no-update async-aws/simple-s3:^0.1.1
61+

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ your issue or pull request in a timely manner, ping us:
4747

4848
| Adapter | Referent |
4949
|--------------------|-----------------------------|
50+
| AsyncAws S3 | @Nyholm |
5051
| AwsS3 | @NiR- |
5152
| AzureBlobStorage | @NiR- |
5253
| DoctrineDbal | @pedrotroller, @NicolasNSSM |

‎appveyor.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ install:
6161
- IF %PHP%==1 echo openssl.cafile=c:\php\cacert.pem >> php.ini
6262
- cd C:\projects\gaufrette
6363
- curl -fsSL -o composer.phar https://getcomposer.org/composer.phar
64-
- php composer.phar require --no-update aws/aws-sdk-php:^3.158 rackspace/php-opencloud:^1.9.2 google/apiclient:^1.1.3 doctrine/dbal:^2.3 league/flysystem:^1.0 microsoft/azure-storage-blob:^1.0 phpseclib/phpseclib:^2.0 mongodb/mongodb:^1.1 symfony/event-dispatcher:^4.4
64+
- php composer.phar require --no-update async-aws/simple-s3:^0.1.1 aws/aws-sdk-php:^3.158 rackspace/php-opencloud:^1.9.2 google/apiclient:^1.1.3 doctrine/dbal:^2.3 league/flysystem:^1.0 microsoft/azure-storage-blob:^1.0 phpseclib/phpseclib:^2.0 mongodb/mongodb:^1.1 symfony/event-dispatcher:^4.4
6565
- php composer.phar update --prefer-dist --no-interaction --no-progress --no-suggest --ansi
6666

6767
test_script:

‎doc/adapters/async-aws-s3.md

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
currentMenu: async-aws-s3
3+
---
4+
5+
# AsyncAws S3
6+
7+
First, you will need to install the simple S3 client:
8+
```bash
9+
composer require async-aws/simple-s3
10+
```
11+
12+
In order to use this adapter you'll need an access key and a secret key.
13+
14+
## Example
15+
16+
```php
17+
<?php
18+
19+
use AsyncAws\SimpleS3\SimpleS3Client;
20+
use Gaufrette\Adapter\AsyncAwsS3 as AwsS3Adapter;
21+
use Gaufrette\Filesystem;
22+
23+
$s3client = new SimpleS3Client([
24+
'accessKeyId' => 'your_key_here',
25+
'accessKeySecret' => 'your_secret',
26+
'region' => 'eu-west-1',
27+
]);
28+
29+
$adapter = new AwsS3Adapter($s3client, 'your-bucket-name');
30+
$filesystem = new Filesystem($adapter);
31+
```
+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?php
2+
3+
namespace spec\Gaufrette\Adapter;
4+
5+
use AsyncAws\SimpleS3\SimpleS3Client;
6+
use Gaufrette\Adapter\MimeTypeProvider;
7+
use PhpSpec\ObjectBehavior;
8+
9+
class AsyncAwsS3Spec extends ObjectBehavior
10+
{
11+
/**
12+
* @param \AsyncAws\SimpleS3\SimpleS3Client $service
13+
*/
14+
function let(SimpleS3Client $service)
15+
{
16+
$this->beConstructedWith($service, 'bucketName');
17+
}
18+
19+
function it_is_initializable()
20+
{
21+
$this->shouldHaveType('Gaufrette\Adapter\AsyncAwsS3');
22+
}
23+
24+
function it_is_adapter()
25+
{
26+
$this->shouldHaveType('Gaufrette\Adapter');
27+
}
28+
29+
function it_supports_metadata()
30+
{
31+
$this->shouldHaveType('Gaufrette\Adapter\MetadataSupporter');
32+
}
33+
34+
function it_supports_sizecalculator()
35+
{
36+
$this->shouldHaveType('Gaufrette\Adapter\SizeCalculator');
37+
}
38+
39+
function it_provides_mime_type()
40+
{
41+
$this->shouldHaveType(MimeTypeProvider::class);
42+
}
43+
}

‎src/Gaufrette/Adapter/AsyncAwsS3.php

+339
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,339 @@
1+
<?php
2+
3+
namespace Gaufrette\Adapter;
4+
5+
use AsyncAws\SimpleS3\SimpleS3Client;
6+
use Gaufrette\Adapter;
7+
use Gaufrette\Util;
8+
9+
/**
10+
* Amazon S3 adapter using the AsyncAws.
11+
*
12+
* @author Michael Dowling <mtdowling@gmail.com>
13+
* @author Tobias Nyholm <tobias.nyholm@gmail.com>
14+
*/
15+
class AsyncAwsS3 implements Adapter, MetadataSupporter, ListKeysAware, SizeCalculator, MimeTypeProvider
16+
{
17+
/** @var SimpleS3Client */
18+
protected $service;
19+
/** @var string */
20+
protected $bucket;
21+
/** @var array */
22+
protected $options;
23+
/** @var bool */
24+
protected $bucketExists;
25+
/** @var array */
26+
protected $metadata = [];
27+
/** @var bool */
28+
protected $detectContentType;
29+
30+
/**
31+
* @param SimpleS3Client $service
32+
* @param string $bucket
33+
* @param array $options
34+
* @param bool $detectContentType
35+
*/
36+
public function __construct(SimpleS3Client $service, $bucket, array $options = [], $detectContentType = false)
37+
{
38+
if (!class_exists(SimpleS3Client::class)) {
39+
throw new \LogicException('You need to install package "async-aws/simple-s3" to use this adapter');
40+
}
41+
$this->service = $service;
42+
$this->bucket = $bucket;
43+
$this->options = array_replace(
44+
[
45+
'create' => false,
46+
'directory' => '',
47+
'acl' => 'private',
48+
],
49+
$options
50+
);
51+
52+
$this->detectContentType = $detectContentType;
53+
}
54+
55+
/**
56+
* {@inheritdoc}
57+
*/
58+
public function setMetadata($key, $content)
59+
{
60+
// BC with AmazonS3 adapter
61+
if (isset($content['contentType'])) {
62+
$content['ContentType'] = $content['contentType'];
63+
unset($content['contentType']);
64+
}
65+
66+
$this->metadata[$key] = $content;
67+
}
68+
69+
/**
70+
* {@inheritdoc}
71+
*/
72+
public function getMetadata($key)
73+
{
74+
return isset($this->metadata[$key]) ? $this->metadata[$key] : [];
75+
}
76+
77+
/**
78+
* {@inheritdoc}
79+
*/
80+
public function read($key)
81+
{
82+
$this->ensureBucketExists();
83+
$options = $this->getOptions($key);
84+
85+
try {
86+
// Get remote object
87+
$object = $this->service->getObject($options);
88+
// If there's no metadata array set up for this object, set it up
89+
if (!array_key_exists($key, $this->metadata) || !is_array($this->metadata[$key])) {
90+
$this->metadata[$key] = [];
91+
}
92+
// Make remote ContentType metadata available locally
93+
$this->metadata[$key]['ContentType'] = $object->getContentType();
94+
95+
return $object->getBody()->getContentAsString();
96+
} catch (\Exception $e) {
97+
return false;
98+
}
99+
}
100+
101+
/**
102+
* {@inheritdoc}
103+
*/
104+
public function rename($sourceKey, $targetKey)
105+
{
106+
$this->ensureBucketExists();
107+
$options = $this->getOptions(
108+
$targetKey,
109+
['CopySource' => $this->bucket . '/' . $this->computePath($sourceKey)]
110+
);
111+
112+
try {
113+
$this->service->copyObject(array_merge($options, $this->getMetadata($targetKey)));
114+
115+
return $this->delete($sourceKey);
116+
} catch (\Exception $e) {
117+
return false;
118+
}
119+
}
120+
121+
/**
122+
* {@inheritdoc}
123+
* @param string|resource $content
124+
*/
125+
public function write($key, $content)
126+
{
127+
$this->ensureBucketExists();
128+
$options = $this->getOptions($key);
129+
unset($options['Bucket'], $options['Key']);
130+
131+
/*
132+
* If the ContentType was not already set in the metadata, then we autodetect
133+
* it to prevent everything being served up as binary/octet-stream.
134+
*/
135+
if (!isset($options['ContentType']) && $this->detectContentType) {
136+
$options['ContentType'] = $this->guessContentType($content);
137+
}
138+
139+
try {
140+
$this->service->upload($this->bucket, $this->computePath($key), $content, $options);
141+
142+
if (is_resource($content)) {
143+
return (int) Util\Size::fromResource($content);
144+
}
145+
146+
return Util\Size::fromContent($content);
147+
} catch (\Exception $e) {
148+
return false;
149+
}
150+
}
151+
152+
/**
153+
* {@inheritdoc}
154+
*/
155+
public function exists($key)
156+
{
157+
return $this->service->has($this->bucket, $this->computePath($key));
158+
}
159+
160+
/**
161+
* {@inheritdoc}
162+
*/
163+
public function mtime($key)
164+
{
165+
try {
166+
$result = $this->service->headObject($this->getOptions($key));
167+
168+
return $result->getLastModified()->getTimestamp();
169+
} catch (\Exception $e) {
170+
return false;
171+
}
172+
}
173+
174+
/**
175+
* {@inheritdoc}
176+
*/
177+
public function size($key)
178+
{
179+
$result = $this->service->headObject($this->getOptions($key));
180+
181+
return (int) $result->getContentLength();
182+
}
183+
184+
public function mimeType($key)
185+
{
186+
$result = $this->service->headObject($this->getOptions($key));
187+
188+
return $result->getContentType();
189+
}
190+
191+
/**
192+
* {@inheritdoc}
193+
*/
194+
public function keys()
195+
{
196+
return $this->listKeys();
197+
}
198+
199+
/**
200+
* {@inheritdoc}
201+
*/
202+
public function listKeys($prefix = '')
203+
{
204+
$this->ensureBucketExists();
205+
206+
$options = ['Bucket' => $this->bucket];
207+
if ((string) $prefix != '') {
208+
$options['Prefix'] = $this->computePath($prefix);
209+
} elseif (!empty($this->options['directory'])) {
210+
$options['Prefix'] = $this->options['directory'];
211+
}
212+
213+
$keys = [];
214+
$result = $this->service->listObjectsV2($options);
215+
foreach ($result->getContents() as $file) {
216+
$keys[] = $this->computeKey($file->getKey());
217+
}
218+
219+
return $keys;
220+
}
221+
222+
/**
223+
* {@inheritdoc}
224+
*/
225+
public function delete($key)
226+
{
227+
try {
228+
$this->service->deleteObject($this->getOptions($key));
229+
230+
return true;
231+
} catch (\Exception $e) {
232+
return false;
233+
}
234+
}
235+
236+
/**
237+
* {@inheritdoc}
238+
*/
239+
public function isDirectory($key)
240+
{
241+
$result = $this->service->listObjectsV2([
242+
'Bucket' => $this->bucket,
243+
'Prefix' => rtrim($this->computePath($key), '/') . '/',
244+
'MaxKeys' => 1,
245+
]);
246+
247+
foreach ($result->getContents(true) as $file) {
248+
return true;
249+
}
250+
251+
return false;
252+
}
253+
254+
/**
255+
* Ensures the specified bucket exists. If the bucket does not exists
256+
* and the create option is set to true, it will try to create the
257+
* bucket. The bucket is created using the same region as the supplied
258+
* client object.
259+
*
260+
* @throws \RuntimeException if the bucket does not exists or could not be
261+
* created
262+
*/
263+
protected function ensureBucketExists()
264+
{
265+
if ($this->bucketExists) {
266+
return true;
267+
}
268+
269+
if ($this->bucketExists = $this->service->bucketExists(['Bucket' => $this->bucket])->isSuccess()) {
270+
return true;
271+
}
272+
273+
if (!$this->options['create']) {
274+
throw new \RuntimeException(sprintf(
275+
'The configured bucket "%s" does not exist.',
276+
$this->bucket
277+
));
278+
}
279+
280+
$this->service->createBucket([
281+
'Bucket' => $this->bucket,
282+
]);
283+
$this->bucketExists = true;
284+
285+
return true;
286+
}
287+
288+
protected function getOptions($key, array $options = [])
289+
{
290+
$options['ACL'] = $this->options['acl'];
291+
$options['Bucket'] = $this->bucket;
292+
$options['Key'] = $this->computePath($key);
293+
294+
/*
295+
* Merge global options for adapter, which are set in the constructor, with metadata.
296+
* Metadata will override global options.
297+
*/
298+
$options = array_merge($this->options, $options, $this->getMetadata($key));
299+
300+
return $options;
301+
}
302+
303+
protected function computePath($key)
304+
{
305+
if (empty($this->options['directory'])) {
306+
return $key;
307+
}
308+
309+
return sprintf('%s/%s', $this->options['directory'], $key);
310+
}
311+
312+
/**
313+
* Computes the key from the specified path.
314+
*
315+
* @param string $path
316+
*
317+
* return string
318+
*/
319+
protected function computeKey($path)
320+
{
321+
return ltrim(substr($path, strlen($this->options['directory'])), '/');
322+
}
323+
324+
/**
325+
* @param string|resource $content
326+
*
327+
* @return string
328+
*/
329+
private function guessContentType($content)
330+
{
331+
$fileInfo = new \finfo(FILEINFO_MIME_TYPE);
332+
333+
if (is_resource($content)) {
334+
return $fileInfo->file(stream_get_meta_data($content)['uri']);
335+
}
336+
337+
return $fileInfo->buffer($content);
338+
}
339+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
<?php
2+
3+
namespace Gaufrette\Functional\Adapter;
4+
5+
use AsyncAws\SimpleS3\SimpleS3Client;
6+
use Gaufrette\Adapter\AsyncAwsS3;
7+
use Gaufrette\Adapter\AwsS3;
8+
use Gaufrette\Filesystem;
9+
10+
class AsyncAwsS3Test extends FunctionalTestCase
11+
{
12+
/** @var string */
13+
private $bucket;
14+
15+
/** @var SimpleS3Client */
16+
private $client;
17+
18+
protected function setUp()
19+
{
20+
$key = getenv('AWS_KEY');
21+
$secret = getenv('AWS_SECRET');
22+
23+
if (empty($key) || empty($secret)) {
24+
$this->markTestSkipped('Either AWS_KEY and/or AWS_SECRET env variables are not defined.');
25+
}
26+
27+
$this->bucket = uniqid(getenv('AWS_BUCKET'));
28+
$this->client = new SimpleS3Client([
29+
'region' => 'eu-west-1',
30+
'accessKeyId' => $key,
31+
'accessKeySecret' => $secret,
32+
]);
33+
34+
$this->createFilesystem(['create' => true]);
35+
}
36+
37+
protected function tearDown()
38+
{
39+
if ($this->client === null) {
40+
return;
41+
}
42+
43+
try {
44+
$this->client->deleteBucket(['Bucket' => $this->bucket]);
45+
} catch (\Throwable $e) {
46+
}
47+
}
48+
49+
private function createFilesystem(array $adapterOptions = [])
50+
{
51+
$this->filesystem = new Filesystem(new AsyncAwsS3($this->client, $this->bucket, $adapterOptions));
52+
}
53+
54+
/**
55+
* @test
56+
* @expectedException \RuntimeException
57+
*/
58+
public function shouldThrowExceptionIfBucketMissingAndNotCreating()
59+
{
60+
$this->createFilesystem();
61+
$this->filesystem->read('foo');
62+
}
63+
64+
/**
65+
* @test
66+
*/
67+
public function shouldWriteObjects()
68+
{
69+
$this->assertEquals(7, $this->filesystem->write('foo', 'testing'));
70+
}
71+
72+
/**
73+
* @test
74+
*/
75+
public function shouldCheckForObjectExistence()
76+
{
77+
$this->filesystem->write('foo', '');
78+
$this->assertTrue($this->filesystem->has('foo'));
79+
}
80+
81+
/**
82+
* @test
83+
*/
84+
public function shouldCheckForObjectExistenceWithDirectory()
85+
{
86+
$this->createFilesystem(['directory' => 'bar', 'create' => true]);
87+
$this->filesystem->write('foo', '');
88+
89+
$this->assertTrue($this->filesystem->has('foo'));
90+
}
91+
92+
/**
93+
* @test
94+
*/
95+
public function shouldListKeysWithoutDirectory()
96+
{
97+
$this->assertEquals([], $this->filesystem->listKeys());
98+
$this->filesystem->write('test.txt', 'some content');
99+
$this->assertEquals(['test.txt'], $this->filesystem->listKeys());
100+
}
101+
102+
/**
103+
* @test
104+
*/
105+
public function shouldListKeysWithDirectory()
106+
{
107+
$this->createFilesystem(['create' => true, 'directory' => 'root/']);
108+
$this->filesystem->write('test.txt', 'some content');
109+
$this->assertEquals(['test.txt'], $this->filesystem->listKeys());
110+
$this->assertTrue($this->filesystem->has('test.txt'));
111+
}
112+
113+
/**
114+
* @test
115+
*/
116+
public function shouldGetKeysWithoutDirectory()
117+
{
118+
$this->filesystem->write('test.txt', 'some content');
119+
$this->assertEquals(['test.txt'], $this->filesystem->keys());
120+
}
121+
122+
/**
123+
* @test
124+
*/
125+
public function shouldGetKeysWithDirectory()
126+
{
127+
$this->createFilesystem(['create' => true, 'directory' => 'root/']);
128+
$this->filesystem->write('test.txt', 'some content');
129+
$this->assertEquals(['test.txt'], $this->filesystem->keys());
130+
}
131+
132+
/**
133+
* @test
134+
*/
135+
public function shouldUploadWithGivenContentType()
136+
{
137+
/** @var AwsS3 $adapter */
138+
$adapter = $this->filesystem->getAdapter();
139+
140+
$adapter->setMetadata('foo', ['ContentType' => 'text/html']);
141+
$this->filesystem->write('foo', '<html></html>');
142+
143+
$this->assertEquals('text/html', $this->filesystem->mimeType('foo'));
144+
}
145+
}

0 commit comments

Comments
 (0)
Please sign in to comment.