Merge branch 'develop' into bugfix/1346-part-2

.ddev/providers/README.txt

@@ -7,23 +7,23 @@ Providers README
 
 DDEV's hosting provider integration lets you integrate with any upstream source of database dumps and files (such as your production or staging server) and provides examples of configuration for Acquia, Platform.sh, Pantheon, rsync, etc.
 
-The best part of this is you can change them and adapt them in any way you need to, they're all short scripted recipes. There are several example recipes created in the .ddev/providers directory of every project or see them in the code at https://github.com/ddev/ddev/tree/master/cmd/ddev/cmd/dotddev_assets/providers.
+The best part of this is you can change them and adapt them in any way you need to, they're all short scripted recipes. There are several example recipes created in the .ddev/providers directory of every project or see them in the code at https://github.com/ddev/ddev/tree/master/pkg/ddevapp/dotddev_assets/providers.
 
 ddev provides the `pull` command with whatever recipes you have configured. For example, `ddev pull acquia` if you have created `.ddev/providers/acquia.yaml`.
 
 ddev also provides the `push` command to push database and files to upstream. This is very dangerous to your upstream site and should only be used with extreme caution. It's recommended not even to implement the push stanzas in your yaml file, but if it fits your workflow, use it well.
 
 Each provider recipe is a yaml file that can be named any way you want to name it. The examples are mostly named after the hosting providers, but they could be named "upstream.yaml" or "live.yaml", so you could `ddev pull upstream` or `ddev pull live`. If you wanted different upstream environments to pull from, you could name one "prod" and one "dev" and `ddev pull prod` and `ddev pull dev`.
 
-Several example recipes are at https://github.com/ddev/ddev/tree/master/cmd/ddev/cmd/dotddev_assets/providers and in this directory.
+Several example recipes are at https://github.com/ddev/ddev/tree/master/pkg/ddevapp/dotddev_assets/providers and in this directory.
 
 Each provider recipe is a file named `<provider>.yaml` and consists of several mostly-optional stanzas:
 
 * `environment_variables`: Environment variables will be created in the web container for each of these during pull or push operations. They're used to provide context (project id, environment name, etc.) for each of the other stanzas.
 * `db_pull_command`: A script that determines how ddev should pull a database. It's job is to create a gzipped database dump in /var/www/html/.ddev/.downloads/db.sql.gz.
 * `files_pull_command`: A script that determines how ddev can get user-generated files from upstream. Its job is to copy the files from upstream to  /var/www/html/.ddev/.downloads/files.
 * `db_push_command`: A script that determines how ddev should push a database. It's job is to take a  gzipped database dump from /var/www/html/.ddev/.downloads/db.sql.gz and load it on the hosting provider.
-* `files_pull_command`: A script that determines how ddev push user-generated files to upstream. Its job is to copy the files from the project's user-files directory ($DDEV_FILES_DIR) to the correct place on the upstream provider.
+* `files_pull_command`: A script that determines how ddev push user-generated files to upstream. Its job is to copy the files from the project's user-files directories ($DDEV_FILES_DIRS) to the correct places on the upstream provider.
 
 The environment variables provided to custom commands (see https://ddev.readthedocs.io/en/stable/users/extend/custom-commands/#environment-variables-provided) are also available for use in these recipes.
 

.ddev/providers/git.yaml.example

@@ -30,11 +30,11 @@ db_import_command:
   command: |
     set -eu -o pipefail
     # set -x
-    ddev import-db --src="${checkout_dir}/db.sql.gz"
+    ddev import-db --file="${checkout_dir}/db.sql.gz"
 
 files_import_command:
   service: host
   command: |
     set -eu -o pipefail
     # set -x
-    ddev import-files --src="${checkout_dir}/files"
+    ddev import-files --source="${checkout_dir}/files"

.ddev/providers/pantheon.yaml.example

@@ -18,7 +18,7 @@
 #
 # 3. On the pantheon dashboard, make sure that at least one backup has been created. (When you need to refresh what you pull, do a new backup.)
 #
-# 4. Make sure your public ssh key is configured in Pantheon (Account->SSH Keys)
+# 4. For `ddev push pantheon` sure your public ssh key is configured in Pantheon (Account->SSH Keys)
 #
 # 5. Check out project codebase from Pantheon. Enable the "Git Connection Mode" and use `git clone` to check out the code locally.
 #
@@ -46,7 +46,6 @@ environment_variables:
 auth_command:
   command: |
     set -eu -o pipefail
-    ssh-add -l >/dev/null || ( echo "Please 'ddev auth ssh' before running this command." && exit 1 )
     if ! command -v drush >/dev/null ; then echo "Please make sure your project contains drush, ddev composer require drush/drush" && exit 1; fi
     if [ -z "${TERMINUS_MACHINE_TOKEN:-}" ]; then echo "Please make sure you have set TERMINUS_MACHINE_TOKEN in ~/.ddev/global_config.yaml" && exit 1; fi
     terminus auth:login --machine-token="${TERMINUS_MACHINE_TOKEN}" || ( echo "terminus auth login failed, check your TERMINUS_MACHINE_TOKEN" && exit 1 )
@@ -73,6 +72,7 @@ files_pull_command:
 db_push_command:
   command: |
     set -x   # You can enable bash debugging output by uncommenting
+    ssh-add -l >/dev/null || ( echo "Please 'ddev auth ssh' before running this command." && exit 1 )
     set -eu -o pipefail
     ls /var/www/html/.ddev >/dev/null # This just refreshes stale NFS if possible
     pushd /var/www/html/.ddev/.downloads >/dev/null;
@@ -83,6 +83,7 @@ db_push_command:
 files_push_command:
   command: |
     set -x   # You can enable bash debugging output by uncommenting
+    ssh-add -l >/dev/null || ( echo "Please 'ddev auth ssh' before running this command." && exit 1 )
     set -eu -o pipefail
     ls ${DDEV_FILES_DIR} >/dev/null # This just refreshes stale NFS if possible
     drush rsync -y @self:%files @${project}:%files

.ddev/providers/platform.yaml

@@ -1,5 +1,7 @@
 #ddev-generated
-# Example Platform.sh provider configuration.
+# Platform.sh provider configuration. This works out of the box, but can be edited to add
+# your own preferences. If you edit it, remove the `#ddev-generated` line from the top so
+# that it won't be overwritten.
 
 # Consider using `ddev get ddev/ddev-platformsh` (https://github.com/ddev/ddev-platformsh) for more
 # complete platform integration.

.gitignore

@@ -35,3 +35,4 @@ tests/.env
 /tests/_craft/storage
 /.sass-cache
 /tests/_craft/.env
+/src/web/assets/feedme/.env

CHANGELOG.md

@@ -2,11 +2,18 @@
 
 ## Unreleased
 
+- Added the ability to provide a filename for importing assets via the assets field if you choose to “Create asset from URL”. ([#749](https://github.com/craftcms/feed-me/issues/749))
 - Fixed a bug where you could not override feed settings with `false` if you used a `config/feed-me.php` config file. ([#1352](https://github.com/craftcms/feed-me/issues/1352))
 - Fixed a bug where `queueTtr`, `queueMaxRetry`, and `assetDownloadCurl` settings were being ignored if you used a `config/feed-me.php` config file. ([#1356](https://github.com/craftcms/feed-me/issues/1356))
 - Fixed a PHP error that could occur when you map an entry field with a default value and there is an empty value in the feed. ([#1250](https://github.com/craftcms/feed-me/pull/1250))
 - Fixed a bug where the `maxRelations` setting was not being respected for relational fields. ([#1354](https://github.com/craftcms/feed-me/issues/1354))
-- Part 2 of ([1346](https://github.com/craftcms/feed-me/pull/1346)). Only cater for Categories and Tags fields.
+- Fixed a bug where importing assets with “Use existing assets” could create duplicate assets. ([#1348](https://github.com/craftcms/feed-me/issues/1348))
+- Fixed a bug where importing Address data from the Google Maps plugin could cause duplicate Addresses if the Address already existed. ([#1370](https://github.com/craftcms/feed-me/pull/1370))
+- Fixed a bug where updating entry statuses for a site in a site group, would update statuses for all sites in the site group instead of the targeted one. ([#1208](https://github.com/craftcms/feed-me/issues/1208))
+- Fixed a bug where you could not use `false` to override feed settings in `config/feed-me.php`. ([#1380](https://github.com/craftcms/feed-me/issues/1380))
+- Fixed a bug where importing nested element fields would not work in some scenarios. ([#1378](https://github.com/craftcms/feed-me/issues/1378))
+- Fixed a bug where empty column headings in a feed would cause incorrect values to be selected in the pagination URL dropdown when mapping a feed. ([#1375](https://github.com/craftcms/feed-me/issues/1375))
+- Fixed a bug that could occur when using Feed Me with fields that have been entrified in Craft. ([1346](https://github.com/craftcms/feed-me/pull/1346)).
 
 ## 5.2.0 - 2023-07-06
 

src/base/Field.php

@@ -167,7 +167,8 @@ protected function populateElementFields($elementIds, $nodeKey = null): void
 
                 // Arrayed content doesn't provide defaults because it's unable to determine how many items it _should_ return
                 // This also checks if there was any data that corresponds on the same array index/level as our element
-                $value = Hash::get($fieldValue, $nodeKey, $default);
+                /** @phpstan-ignore-next-line */
+                $value = Hash::get($fieldValue, $nodeKey ?? $key, $default);
 
                 if ($value) {
                     $fieldData[$elementId][$fieldHandle] = $value;

src/datatypes/Csv.php

@@ -70,9 +70,14 @@ public function getFeed($url, $settings, bool $usePrimaryElement = true): array
                 $filteredRow = [];
 
                 // Additional work here to handle line-breaks in keys (CSV header) - they're not allowed
+                $col = 1;
                 foreach ($row as $key => $value) {
                     $newKey = preg_replace('#\r\n?#', " ", $key);
+                    if (trim($newKey) == '') {
+                        $newKey = 'blank_heading_' . $col;
+                    }
                     $filteredRow[$newKey] = $value;
+                    $col++;
                 }
 
                 // Check for empty rows - ditch them

src/datatypes/GoogleSheet.php

@@ -55,6 +55,10 @@ public function getFeed($url, $settings, bool $usePrimaryElement = true): array
                 foreach ($row as $j => $column) {
                     $key = $headers[$j];
 
+                    if (trim($key) == '') {
+                        $key = 'blank_heading_' . ($j + 1);
+                    }
+
                     $array[$i][$key] = $column;
                 }
             }

src/datatypes/Json.php

@@ -62,6 +62,19 @@ public function getFeed($url, $settings, bool $usePrimaryElement = true): array
             return ['success' => false, 'error' => $error];
         }
 
+        // if we have empty keys in the array - throw an error, that's not allowed
+        $containsEmptyKeys = false;
+        array_walk_recursive($array, function($value, $key) use (&$containsEmptyKeys) {
+            if (trim($key) === '') {
+                $containsEmptyKeys = true;
+            }
+        });
+        if ($containsEmptyKeys) {
+            $error = 'Invalid Data: data contains empty headings (keys)';
+            Plugin::error($error);
+            return ['success' => false, 'error' => $error];
+        }
+
         // If using pagination, set it up here - we need to do this before messing around with the primary element
         $this->setupPaginationUrl($array, $settings);
 

src/events/AssetFilenameEvent.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace craft\feedme\events;
+
+use yii\base\Event;
+
+class AssetFilenameEvent extends Event
+{
+    // Properties
+    // =========================================================================
+
+    /**
+     * @var
+     */
+    public $field;
+
+    /**
+     * @var
+     */
+    public $element;
+
+    /**
+     * @var
+     */
+    public $fieldValue;
+
+    /**
+     * @var
+     */
+    public $filenames;
+}

src/fields/Assets.php

@@ -9,6 +9,7 @@
 use craft\elements\Asset as AssetElement;
 use craft\feedme\base\Field;
 use craft\feedme\base\FieldInterface;
+use craft\feedme\events\AssetFilenameEvent;
 use craft\feedme\helpers\AssetHelper;
 use craft\feedme\helpers\DataHelper;
 use craft\feedme\Plugin;
@@ -23,6 +24,8 @@
  */
 class Assets extends Field implements FieldInterface
 {
+    public const EVENT_ASSET_FILENAME = 'onAssetFilename';
+
     // Properties
     // =========================================================================
 
@@ -113,6 +116,21 @@ public function parseField(): mixed
         $urlsToUpload = [];
         $base64ToUpload = [];
 
+        $filenamesFromFeed = $upload ? DataHelper::fetchArrayValue($this->feedData, $this->fieldInfo, 'options.filenameNode') : null;
+
+        // Fire an 'onAssetFilename' event
+        $event = new AssetFilenameEvent([
+            'field' => $this->field,
+            'element' => $this->element,
+            'fieldValue' => $value,
+            'filenames' => $filenamesFromFeed,
+        ]);
+
+        $this->trigger(self::EVENT_ASSET_FILENAME, $event);
+
+        // Allow event to overwrite filenames to be used
+        $filenamesFromFeed = $event->filenames;
+
         foreach ($value as $key => $dataValue) {
             // Prevent empty or blank values (string or array), which match all elements
             if (empty($dataValue) && empty($default)) {
@@ -155,8 +173,15 @@ public function parseField(): mixed
                 // If we're uploading files, this will need to be an absolute URL. If it is, save until later.
                 // We also don't check for existing assets here, so break out instantly.
                 if ($upload && UrlHelper::isAbsoluteUrl($dataValue)) {
-                    $urlsToUpload[$key] = $dataValue;
-                    $filename = AssetHelper::getRemoteUrlFilename($dataValue);
+                    $urlsToUpload[$key]['value'] = $dataValue;
+
+                    if (isset($filenamesFromFeed[$key])) {
+                        $filename = $filenamesFromFeed[$key] . '.' . AssetHelper::getRemoteUrlExtension($urlsToUpload[$key]['value']);
+                        $urlsToUpload[$key]['newFilename'] = $filename;
+                    } else {
+                        $filename = AssetHelper::getRemoteUrlFilename($dataValue);
+                        $urlsToUpload[$key]['newFilename'] = null;
+                    }
                 } else {
                     $filename = basename($dataValue);
                 }
@@ -190,8 +215,18 @@ public function parseField(): mixed
 
         if ($upload) {
             if ($urlsToUpload) {
-                $uploadedElements = AssetHelper::fetchRemoteImage($urlsToUpload, $this->fieldInfo, $this->feed, $this->field, $this->element);
-                $foundElements = array_merge($foundElements, $uploadedElements);
+                foreach ($urlsToUpload as $item) {
+                    $uploadedElements = AssetHelper::fetchRemoteImage(
+                        [$item['value']],
+                        $this->fieldInfo,
+                        $this->feed,
+                        $this->field,
+                        $this->element,
+                        null,
+                        $item['newFilename']
+                    );
+                    $foundElements = array_merge($foundElements, $uploadedElements);
+                }
             }
 
             if ($base64ToUpload) {

src/fields/SuperTable.php

@@ -101,7 +101,7 @@ public function parseField(): mixed
 
                 // Finish up with the content, also sort out cases where there's array content
                 if (isset($fieldData[$key]) && is_array($fieldData[$key])) {
-                    $fieldData[$key] = array_merge_recursive($fieldData[$key], $parsedValue);
+                    $fieldData[$key] = is_array($parsedValue) ? array_merge_recursive($fieldData[$key], $parsedValue) : $fieldData[$key];
                 } else {
                     $fieldData[$key] = $parsedValue;
                 }
@@ -122,7 +122,7 @@ public function parseField(): mixed
             $parsedValue = $this->_parseSubField($nodePaths, $subFieldHandle, $subFieldInfo);
 
             if (isset($fieldData[$key])) {
-                $fieldData[$key] = array_merge_recursive($fieldData[$key], $parsedValue);
+                $fieldData[$key] = is_array($parsedValue) ? array_merge_recursive($fieldData[$key], $parsedValue) : $fieldData[$key];
             } else {
                 $fieldData[$key] = $parsedValue;
             }