From 7a642f287cac708a29b3d1a0417eb79ac169dac6 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Mon, 11 Jun 2018 22:55:53 +0200 Subject: [PATCH 001/116] Handle Azure Cosmos DB connections When attempting to connect with an Azure Cosmos DB instance, `can-connect` for Mongo driver returns false due to a mismatch in returned data types. Mongo DB instances using wire protocol versions prior to 3.4 return an an integer value (1) as the value for the "ok" field in the response for db.stats(). Newer versions respond with a float value (1.0). Azure Cosmos DB uses wire protocol version 3.2 by default. This changeset casts the response value to float to ensure expected behaviour. Resolves: #5571 --- src/metabase/driver/mongo.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metabase/driver/mongo.clj b/src/metabase/driver/mongo.clj index 5296b2d0db7ec..493b4c2715ab6 100644 --- a/src/metabase/driver/mongo.clj +++ b/src/metabase/driver/mongo.clj @@ -23,9 +23,9 @@ (defn- can-connect? [details] (with-mongo-connection [^DB conn, details] - (= (-> (cmd/db-stats conn) - (conv/from-db-object :keywordize) - :ok) + (= (float (-> (cmd/db-stats conn) + (conv/from-db-object :keywordize) + :ok)) 1.0))) (defn- humanize-connection-error-message [message] From 92a7e4368a43cb3dc79c2d7011be6fa0d6d2c6ba Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Wed, 20 Jun 2018 17:10:56 -0700 Subject: [PATCH 002/116] Misc QueryHeader cleanup --- .../query_builder/components/QueryHeader.jsx | 61 ++++++------------- 1 file changed, 19 insertions(+), 42 deletions(-) diff --git a/frontend/src/metabase/query_builder/components/QueryHeader.jsx b/frontend/src/metabase/query_builder/components/QueryHeader.jsx index 00d9bda0ce303..0127fc7201886 100644 --- a/frontend/src/metabase/query_builder/components/QueryHeader.jsx +++ b/frontend/src/metabase/query_builder/components/QueryHeader.jsx @@ -22,9 +22,8 @@ import SaveQuestionModal from "metabase/containers/SaveQuestionModal.jsx"; import { clearRequestState } from "metabase/redux/requests"; -import { CardApi, RevisionApi } from "metabase/services"; +import { RevisionApi } from "metabase/services"; -import MetabaseAnalytics from "metabase/lib/analytics"; import * as Urls from "metabase/lib/urls"; import cx from "classnames"; @@ -62,21 +61,6 @@ export default class QueryHeader extends Component { modal: null, revisions: null, }; - - _.bindAll( - this, - "resetStateOnTimeout", - "onCreate", - "onSave", - "onBeginEditing", - "onCancel", - "onDelete", - "onFollowBreadcrumb", - "onToggleDataReference", - "onFetchRevisions", - "onRevertToRevision", - "onRevertedRevision", - ); } static propTypes = { @@ -98,14 +82,14 @@ export default class QueryHeader extends Component { clearTimeout(this.timeout); } - resetStateOnTimeout() { + resetStateOnTimeout = () => { // clear any previously set timeouts then start a new one clearTimeout(this.timeout); this.timeout = setTimeout( () => this.setState({ recentlySaved: null }), 5000, ); - } + }; onCreate = async (card, showSavedModal = true) => { const { question, apiCreateQuestion } = this.props; @@ -140,52 +124,45 @@ export default class QueryHeader extends Component { ); }; - onBeginEditing() { + onBeginEditing = () => { this.props.onBeginEditing(); - } + }; - async onCancel() { + onCancel = async () => { if (this.props.fromUrl) { this.onGoBack(); } else { this.props.onCancelEditing(); } - } - - async onDelete() { - // TODO: reduxify - await CardApi.delete({ cardId: this.props.card.id }); - this.onGoBack(); - MetabaseAnalytics.trackEvent("QueryBuilder", "Delete"); - } + }; - onFollowBreadcrumb() { + onFollowBreadcrumb = () => { this.props.onRestoreOriginalQuery(); - } + }; - onToggleDataReference() { + onToggleDataReference = () => { this.props.toggleDataReferenceFn(); - } + }; - onGoBack() { + onGoBack = () => { this.props.onChangeLocation(this.props.fromUrl || "/"); - } + }; - async onFetchRevisions({ entity, id }) { + onFetchRevisions = async ({ entity, id }) => { // TODO: reduxify let revisions = await RevisionApi.list({ entity, id }); this.setState({ revisions }); - } + }; - onRevertToRevision({ entity, id, revision_id }) { + onRevertToRevision = ({ entity, id, revision_id }) => { // TODO: reduxify return RevisionApi.revert({ entity, id, revision_id }); - } + }; - onRevertedRevision() { + onRevertedRevision = () => { this.props.reloadCardFn(); this.refs.cardHistory.toggle(); - } + }; getHeaderButtons() { const { From fc56c0254ba5b5425b22d83efdf006efd4e7f31d Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Wed, 20 Jun 2018 17:12:37 -0700 Subject: [PATCH 003/116] Add Question redux{Create,Update} and use in QB to ensure the store is updated when questions are created/saved Also replace archive action with one that uses Questions.actions.setArchived --- frontend/src/metabase-lib/lib/Question.js | 18 +++++++++++++-- .../src/metabase/query_builder/actions.js | 22 +++++-------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/frontend/src/metabase-lib/lib/Question.js b/frontend/src/metabase-lib/lib/Question.js index 48cdca5140953..95cb08682d3c6 100644 --- a/frontend/src/metabase-lib/lib/Question.js +++ b/frontend/src/metabase-lib/lib/Question.js @@ -40,6 +40,8 @@ import type { } from "metabase/meta/types/Card"; import { MetabaseApi, CardApi } from "metabase/services"; +import Questions from "metabase/entities/questions"; + import AtomicQuery from "metabase-lib/lib/queries/AtomicQuery"; import type { Dataset } from "metabase/meta/types/Dataset"; @@ -471,16 +473,28 @@ export default class Question { } } + // NOTE: prefer `reduxCreate` so the store is automatically updated async apiCreate() { - const createdCard = await CardApi.create(this.card()); + const createdCard = await Questions.api.create(this.card()); return this.setCard(createdCard); } + // NOTE: prefer `reduxUpdate` so the store is automatically updated async apiUpdate() { - const updatedCard = await CardApi.update(this.card()); + const updatedCard = await Questions.api.update(this.card()); return this.setCard(updatedCard); } + async reduxCreate(dispatch) { + const { payload } = await dispatch(Questions.actions.create(this.card())); + return this.setCard(payload.entities.questions[payload.result]); + } + + async reduxUpdate(dispatch) { + const { payload } = await dispatch(Questions.actions.update(this.card())); + return this.setCard(payload.entities.questions[payload.result]); + } + // TODO: Fix incorrect Flow signature parameters(): ParameterObject[] { return getParametersWithExtras(this.card(), this._parameterValues); diff --git a/frontend/src/metabase/query_builder/actions.js b/frontend/src/metabase/query_builder/actions.js index 78b4490c4f8f2..df318d9759a3a 100644 --- a/frontend/src/metabase/query_builder/actions.js +++ b/frontend/src/metabase/query_builder/actions.js @@ -28,7 +28,6 @@ import { isPK } from "metabase/lib/types"; import Utils from "metabase/lib/utils"; import { getEngineNativeType, formatJsonQuery } from "metabase/lib/engine"; import { defer } from "metabase/lib/promise"; -import { addUndo } from "metabase/redux/undo"; import Question from "metabase-lib/lib/Question"; import { cardIsEquivalent, cardQueryIsEquivalent } from "metabase/meta/Card"; @@ -66,6 +65,8 @@ import NativeQuery from "metabase-lib/lib/queries/NativeQuery"; import { getPersistableDefaultSettings } from "metabase/visualizations/lib/settings"; import { clearRequestState } from "metabase/redux/requests"; +import Questions from "metabase/entities/questions"; + type UiControls = { isEditing?: boolean, isShowingTemplateTagsEditor?: boolean, @@ -775,7 +776,7 @@ export const apiCreateQuestion = question => { const createdQuestion = await questionWithVizSettings .setQuery(question.query().clean()) .setResultsMetadata(resultsMetadata) - .apiCreate(); + .reduxCreate(dispatch); // remove the databases in the store that are used to populate the QB databases list. // This is done when saving a Card because the newly saved card will be eligible for use as a source query @@ -808,7 +809,7 @@ export const apiUpdateQuestion = question => { const updatedQuestion = await questionWithVizSettings .setQuery(question.query().clean()) .setResultsMetadata(resultsMetadata) - .apiUpdate(); + .reduxUpdate(dispatch); // reload the question alerts for the current question // (some of the old alerts might be removed during update) @@ -1429,22 +1430,11 @@ export const ARCHIVE_QUESTION = "metabase/qb/ARCHIVE_QUESTION"; export const archiveQuestion = createThunkAction( ARCHIVE_QUESTION, (questionId, archived = true) => async (dispatch, getState) => { - let card = { - ...getState().qb.card, // grab the current card - archived, - }; - let response = await CardApi.update(card); + let card = getState().qb.card; - dispatch( - addUndo({ - verb: archived ? "archived" : "unarchived", - subject: "question", - action: archiveQuestion(card.id, !archived), - }), - ); + await dispatch(Questions.actions.setArchived({ id: card.id }, archived)); dispatch(push(Urls.collection(card.collection_id))); - return response; }, ); From 16299395803bcf6c43cfaed52816c80b16cf5efe Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 21 Jun 2018 18:51:15 +0200 Subject: [PATCH 004/116] Improve copy. Make cell titles more sensible --- .../automagic_dashboards/field/Country.yaml | 2 +- .../automagic_dashboards/field/DateTime.yaml | 2 +- .../field/GenericField.yaml | 4 +- .../automagic_dashboards/field/Number.yaml | 8 +- .../automagic_dashboards/field/State.yaml | 2 +- .../metric/GenericMetric.yaml | 10 +- .../table/GenericTable.yaml | 2 +- src/metabase/automagic_dashboards/core.clj | 166 ++++++++++++++---- src/metabase/automagic_dashboards/filters.clj | 6 +- .../automagic_dashboards/populate.clj | 22 +-- 10 files changed, 163 insertions(+), 61 deletions(-) diff --git a/resources/automagic_dashboards/field/Country.yaml b/resources/automagic_dashboards/field/Country.yaml index fa493bb6e66d2..729c0d4e287cb 100644 --- a/resources/automagic_dashboards/field/Country.yaml +++ b/resources/automagic_dashboards/field/Country.yaml @@ -38,7 +38,7 @@ groups: - Overview: title: Overview - Breakdowns: - title: How [[this]] are distributed + title: How the [[this]] is distributed cards: - Count: title: Count diff --git a/resources/automagic_dashboards/field/DateTime.yaml b/resources/automagic_dashboards/field/DateTime.yaml index 8af35916858fe..38ccb0632cf2f 100644 --- a/resources/automagic_dashboards/field/DateTime.yaml +++ b/resources/automagic_dashboards/field/DateTime.yaml @@ -32,7 +32,7 @@ groups: - Overview: title: Overview - Breakdowns: - title: How [[this]] is distributed + title: How the [[this]] is distributed - Seasonality: title: Seasonal patterns in [[this]] cards: diff --git a/resources/automagic_dashboards/field/GenericField.yaml b/resources/automagic_dashboards/field/GenericField.yaml index 71d7143394f25..a5ec4274d7dc9 100644 --- a/resources/automagic_dashboards/field/GenericField.yaml +++ b/resources/automagic_dashboards/field/GenericField.yaml @@ -38,7 +38,7 @@ groups: - Overview: title: Overview - Breakdowns: - title: How [[this]] are distributed + title: How the [[this]] is distributed cards: - Count: title: Count @@ -60,7 +60,7 @@ cards: group: Overview width: 6 - Distribution: - title: How [[this]] is distributed + title: How the [[this]] is distributed visualization: bar metrics: Count dimensions: this diff --git a/resources/automagic_dashboards/field/Number.yaml b/resources/automagic_dashboards/field/Number.yaml index 937e04ec5c0df..01a3864a03780 100644 --- a/resources/automagic_dashboards/field/Number.yaml +++ b/resources/automagic_dashboards/field/Number.yaml @@ -38,11 +38,11 @@ groups: - Overview: title: Overview - Breakdowns: - title: How [[this]] is distributed across categories + title: How the [[this]] is distributed across categories - Seasonality: - title: How [[this]] changes with time + title: How the [[this]] changes with time - Geographical: - title: How [[this]] is distributed geographically + title: How the [[this]] is distributed geographically cards: - Count: title: Count @@ -75,7 +75,7 @@ cards: group: Overview width: 9 - Distribution: - title: How [[this]] is distributed + title: How the [[this]] is distributed visualization: bar metrics: Count dimensions: diff --git a/resources/automagic_dashboards/field/State.yaml b/resources/automagic_dashboards/field/State.yaml index 2df61dca59ed4..f6c19178b715e 100644 --- a/resources/automagic_dashboards/field/State.yaml +++ b/resources/automagic_dashboards/field/State.yaml @@ -38,7 +38,7 @@ groups: - Overview: title: Overview - Breakdowns: - title: How [[this]] are distributed + title: How the [[this]] is distributed cards: - Count: title: Count diff --git a/resources/automagic_dashboards/metric/GenericMetric.yaml b/resources/automagic_dashboards/metric/GenericMetric.yaml index 46287df773328..3c8c22b092e45 100644 --- a/resources/automagic_dashboards/metric/GenericMetric.yaml +++ b/resources/automagic_dashboards/metric/GenericMetric.yaml @@ -1,4 +1,4 @@ -title: A look at your [[this]] +title: A look at the [[this]] transient_title: Here's a quick look at your [[this]] description: How it's distributed across time and other categories. applies_to: GenericTable @@ -39,9 +39,9 @@ groups: - Geographical: title: "[[this]] by location" - Categories: - title: How [[this]] is distributed across different categories + title: How the [[this]] is distributed across different categories - Numbers: - title: How [[this]] is distributed across different numbers + title: How the [[this]] is distributed across different numbers - LargeCategories: title: Top and bottom [[this]] dashboard_filters: @@ -121,7 +121,9 @@ cards: group: Numbers title: How [[this]] is distributed across [[GenericNumber]] metrics: this - dimensions: GenericNumber + dimensions: + - GenericNumber: + aggregation: default visualization: bar - ByCategoryMedium: group: Categories diff --git a/resources/automagic_dashboards/table/GenericTable.yaml b/resources/automagic_dashboards/table/GenericTable.yaml index cb9744168ce42..50030336bb768 100644 --- a/resources/automagic_dashboards/table/GenericTable.yaml +++ b/resources/automagic_dashboards/table/GenericTable.yaml @@ -97,7 +97,7 @@ groups: - Geographical: title: Where your [[this]] are - General: - title: How [[this]] are distributed + title: How [[this]] is distributed dashboard_filters: - Timestamp - Date diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 844cf3725b248..815501597da25 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -56,16 +56,32 @@ field/map->FieldInstance (classify/run-classifiers {}))))) +(def ^:private op->name + {:sum (tru "sum") + :avg (tru "average") + :min (tru "minumum") + :max (tru "maximum") + :count (tru "count") + :distinct (tru "distinct count") + :stddev (tru "standard deviation") + :cum-count (tru "cumulative count") + :cum-sum (tru "cumulative sum")}) + +(defn- metric-name + [[op arg]] + (let [op (qp.util/normalize-token op)] + (if (= op :metric) + (-> arg Metric :name) + (op->name op)))) + (defn- metric->description [root metric] - (let [aggregation-clause (-> metric :definition :aggregation first) - field (some->> aggregation-clause - second - filters/field-reference->id - (->field root))] - (if field - (tru "{0} of {1}" (-> aggregation-clause first name str/capitalize) (:display_name field)) - (-> aggregation-clause first name str/capitalize)))) + (tru "{0} of {1}" (metric-name metric) (or (some->> metric + second + filters/field-reference->id + (->field root) + :display_name) + (-> root :source :name)))) (defn- join-enumeration [[x & xs]] @@ -76,19 +92,7 @@ (defn- question-description [root question] (let [aggregations (->> (qp.util/get-in-normalized question [:dataset_query :query :aggregation]) - (map (fn [[op arg]] - (cond - (-> op qp.util/normalize-token (= :metric)) - (-> arg Metric :name) - - arg - (tru "{0} of {1}" (name op) (->> arg - filters/field-reference->id - (->field root) - :display_name)) - - :else - (name op)))) + (map (partial metric->description root)) join-enumeration) dimensions (->> (qp.util/get-in-normalized question [:dataset_query :query :breakout]) (mapcat filters/collect-field-references) @@ -121,7 +125,7 @@ [segment] (let [table (-> segment :table_id Table)] {:entity segment - :full-name (tru "{0} segment" (:name segment)) + :full-name (tru "{0} in {1} segment" (:display_name table) (:name segment)) :source table :database (:db_id table) :query-filter (-> segment :definition :filter) @@ -184,7 +188,7 @@ :source (source card) :database (:database_id card) :query-filter (qp.util/get-in-normalized card [:dataset_query :query :filter]) - :full-name (tru "{0} question" (:name card)) + :full-name (tru "\"{0}\" question" (:name card)) :url (format "%squestion/%s" public-endpoint (u/get-id card)) :rules-prefix [(if (table-like? card) "table" @@ -489,18 +493,25 @@ (u/update-when :graph.metrics metric->name) (u/update-when :graph.dimensions dimension->name))])) +(defn- capitalize-first + [s] + (str (str/upper-case (subs s 0 1)) (subs s 1))) + (defn- instantiate-metadata [x context bindings] (-> (walk/postwalk (fn [form] (if (string? form) - (fill-templates :string context bindings form) + (let [new-form (fill-templates :string context bindings form)] + (if (not= new-form form) + (capitalize-first new-form) + new-form)) form)) x) (u/update-when :visualization #(instantate-visualization % bindings (:metrics context))))) (defn- valid-breakout-dimension? - [{:keys [base_type engine] :as f}] + [{:keys [base_type engine]}] (not (and (isa? base_type :type/Number) (= engine :druid)))) @@ -542,13 +553,13 @@ limit order_by))] (-> card - (assoc :metrics metrics) (instantiate-metadata context (->> metrics (map :name) (zipmap (:metrics card)) (merge bindings))) - (assoc :score score - :dataset_query query)))))))) + (assoc :dataset_query query + :metrics (map (some-fn :name (comp metric-name :metric)) metrics) + :score score)))))))) (defn- matching-rules "Return matching rules orderd by specificity. @@ -748,6 +759,83 @@ {:related (related-entities n-related-entities root) :drilldown-fields (take (- max-related n-related-entities) drilldown-fields)})))) +(def ^:private date-formatter (t.format/formatter "MMMM d, YYYY")) +(def ^:private datetime-formatter (t.format/formatter "EEEE, MMMM d, YYYY h:mm a")) + +(defn- humanize-datetime + [dt] + (t.format/unparse (if (str/index-of dt "T") + datetime-formatter + date-formatter) + (t.format/parse dt))) + +(defn- field-reference->field + [fieldset field-reference] + (cond-> (-> field-reference + filters/collect-field-references + first + filters/field-reference->id + fieldset) + (-> field-reference first qp.util/normalize-token (= :datetime-field)) + (assoc :unit (-> field-reference last qp.util/normalize-token)))) + +(defmulti + ^{:private true + :arglists '([fieldset [op & args]])} + humanize-filter-value (fn [_ [op & args]] + (qp.util/normalize-token op))) + +(def ^:private unit-name (comp {:minute-of-hour "minute of hour" + :hour-of-day "hour of day" + :day-of-week "day of week" + :day-of-month "day of month" + :week-of-year "week of year" + :month-of-year "month of year" + :quarter-of-year "quarter of year"} + qp.util/normalize-token)) + +(defn- field-name + ([fieldset field-reference] + (->> field-reference (field-reference->field fieldset) field-name)) + ([{:keys [display_name unit] :as field}] + (cond->> display_name + (and (filters/periodic-datetime? field) unit) (format "%s of %s" (unit-name unit))))) + +(defmethod humanize-filter-value := + [fieldset [_ field-reference value]] + (let [field (field-reference->field fieldset field-reference) + field-name (field-name field)] + (cond + (#{:type/State :type/Country} (:special_type field)) + (tru "in {0}" value) + + (filters/datetime? field) + (tru "where {0} is on {1}" field-name (humanize-datetime value)) + + :else + (tru "where {0} is {1}" field-name value)))) + +(defmethod humanize-filter-value :between + [fieldset [_ field-reference min-value max-value]] + (tru "where {0} is between {1} and {2}" (field-name fieldset field-reference) min-value max-value)) + +(defmethod humanize-filter-value :inside + [fieldset [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]] + (tru "where {0} is between {1} and {2}; and {3} is between {4} and {5}" + (field-name fieldset lon-reference) lon-min lon-max + (field-name fieldset lat-reference) lat-min lat-max)) + +(defn- cell-title + [context cell-query] + (let [source-name (-> context :root :source ((some-fn :display_name :name))) + fieldset (->> context + :tables + (mapcat :fields) + (map (fn [field] + [((some-fn :id :name) field) field])) + (into {}))] + (str/join " " [source-name (humanize-filter-value fieldset cell-query)]))) + (defn- automagic-dashboard "Create dashboards for table `root` using the best matching heuristics." [{:keys [rule show rules-prefix query-filter cell-query full-name] :as root}] @@ -772,12 +860,14 @@ (-> dashboard :context :filters u/pprint-to-str)) (-> (cond-> dashboard (or query-filter cell-query) - (assoc :title (tru "A closer look at {0}" full-name))) + (assoc :title (tru "A closer look at {0}" + (cell-title (:context dashboard) cell-query)) + :transient_title nil)) (populate/create-dashboard (or show max-cards)) - (assoc :related (related dashboard rule)) - (assoc :more (when (and (-> dashboard :cards count (> max-cards)) - (not= show :all)) - (format "%s#show=all" (:url root)))))) + (assoc :related (related dashboard rule) + :more (when (and (-> dashboard :cards count (> max-cards)) + (not= show :all)) + (format "%s#show=all" (:url root)))))) (throw (ex-info (trs "Can''t create dashboard for {0}" full-name) {:root root :available-rules (map :rule (or (some-> rule rules/get-rule vector) @@ -813,7 +903,7 @@ {:definition {:aggregation [aggregation-clause] :source_table (:table_id question)} :table_id (:table_id question)})] - (assoc metric :name (metric->description root metric))))) + (assoc metric :name (metric->description root aggregation-clause))))) (qp.util/get-in-normalized question [:dataset_query :query :aggregation]))) (defn- collect-breakout-fields @@ -843,6 +933,10 @@ (u/get-id card) (encode-base64-json cell-query)) :entity (:source root) + :full-name (->> root + :source + ((some-fn :display_name :name)) + (tru "such {0}")) :rules-prefix ["table"]})) opts)) (let [opts (assoc opts :show :all)] @@ -861,6 +955,10 @@ (encode-base64-json (:dataset_query query)) (encode-base64-json cell-query)) :entity (:source root) + :full-name (->> root + :source + ((some-fn :display_name :name)) + (tru "such {0}")) :rules-prefix ["table"]})) (update opts :cell-query (partial filters/inject-refinement diff --git a/src/metabase/automagic_dashboards/filters.clj b/src/metabase/automagic_dashboards/filters.clj index d6d7c5b8c7e63..5f1a3c30ba879 100644 --- a/src/metabase/automagic_dashboards/filters.clj +++ b/src/metabase/automagic_dashboards/filters.clj @@ -44,12 +44,14 @@ (tree-seq (some-fn sequential? map?) identity) (filter field-reference?))) -(def ^:private ^{:arglists '([field])} periodic-datetime? +(def ^{:arglists '([field])} periodic-datetime? + "Is `field` a periodic datetime (eg. day of month)?" (comp #{:minute-of-hour :hour-of-day :day-of-week :day-of-month :day-of-year :week-of-year :month-of-year :quarter-of-year} :unit)) -(defn- datetime? +(defn datetime? + "Is `field` a datetime?" [field] (and (not (periodic-datetime? field)) (or (isa? (:base_type field) :type/DateTime) diff --git a/src/metabase/automagic_dashboards/populate.clj b/src/metabase/automagic_dashboards/populate.clj index fb1c1c048d320..bf671adab3cc2 100644 --- a/src/metabase/automagic_dashboards/populate.clj +++ b/src/metabase/automagic_dashboards/populate.clj @@ -80,17 +80,17 @@ (defn- visualization-settings [{:keys [metrics x_label y_label series_labels visualization dimensions] :as card}] - (let [metric-name (some-fn :name (comp str/capitalize name first :metric)) - [display visualization-settings] visualization] + (let [[display visualization-settings] visualization] {:display display - :visualization_settings - (-> visualization-settings - (merge (colorize card)) - (cond-> - (some :name metrics) (assoc :graph.series_labels (map metric-name metrics)) - series_labels (assoc :graph.series_labels series_labels) - x_label (assoc :graph.x_axis.title_text x_label) - y_label (assoc :graph.y_axis.title_text y_label)))})) + :visualization_settings (-> visualization-settings + (assoc :graph.series_labels metrics) + (merge (colorize card)) + (cond-> + series_labels (assoc :graph.series_labels series_labels) + + x_label (assoc :graph.x_axis.title_text x_label) + + y_label (assoc :graph.y_axis.title_text y_label)))})) (defn- add-card "Add a card to dashboard `dashboard` at position [`x`, `y`]." @@ -236,7 +236,7 @@ (defn create-dashboard "Create dashboard and populate it with cards." ([dashboard] (create-dashboard dashboard :all)) - ([{:keys [title transient_title description groups filters cards refinements fieldset]} n] + ([{:keys [title transient_title description groups filters cards refinements]} n] (let [n (cond (= n :all) (count cards) (keyword? n) (Integer/parseInt (name n)) From 86396d19aa1c13509b044512f3772f882fdab5d1 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 21 Jun 2018 19:02:39 +0200 Subject: [PATCH 005/116] Split cases where we have query-filter and cell-query --- src/metabase/automagic_dashboards/core.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 815501597da25..7124de88ef169 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -859,10 +859,12 @@ (-> dashboard :context :metrics u/pprint-to-str) (-> dashboard :context :filters u/pprint-to-str)) (-> (cond-> dashboard - (or query-filter cell-query) + cell-query (assoc :title (tru "A closer look at {0}" (cell-title (:context dashboard) cell-query)) - :transient_title nil)) + :transient_title nil) + query-filter + (assoc :title (tru "A closer look at {0}" (:full-name root)))) (populate/create-dashboard (or show max-cards)) (assoc :related (related dashboard rule) :more (when (and (-> dashboard :cards count (> max-cards)) From ba31d86c4e645edc6557dc7d6c6e25bef641bb03 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Fri, 22 Jun 2018 11:49:17 +0200 Subject: [PATCH 006/116] Introduce [[this.foo]] syntax and make GenericTable and segments nice --- .../table/GenericTable.yaml | 76 +++++++++---------- .../automagic_dashboards/table/UserTable.yaml | 22 +++--- src/metabase/automagic_dashboards/core.clj | 22 ++++-- 3 files changed, 64 insertions(+), 56 deletions(-) diff --git a/resources/automagic_dashboards/table/GenericTable.yaml b/resources/automagic_dashboards/table/GenericTable.yaml index 50030336bb768..b7dbae520b086 100644 --- a/resources/automagic_dashboards/table/GenericTable.yaml +++ b/resources/automagic_dashboards/table/GenericTable.yaml @@ -91,13 +91,13 @@ groups: - Overview: title: Summary - Singletons: - title: These are the same for all your [[this]] + title: These are the same for all your [[this.short-name]] - ByTime: - title: "[[this]] across time" + title: "[[this.short-name]] across time" - Geographical: - title: Where your [[this]] are + title: Where your [[this.short-name]] are - General: - title: How [[this]] is distributed + title: How [[this.short-name]] are distributed dashboard_filters: - Timestamp - Date @@ -112,13 +112,13 @@ dashboard_filters: cards: # Overview - Rowcount: - title: Total [[this]] + title: Total [[this.short-name]] visualization: scalar metrics: Count score: 100 group: Overview - RowcountLast30Days: - title: New [[this]] in the last 30 days + title: New [[this.short-name]] in the last 30 days visualization: scalar metrics: Count score: 100 @@ -132,7 +132,7 @@ cards: group: Overview # General - NumberDistribution: - title: How [[this]] are distributed across [[GenericNumber]] + title: How [[this.short-name]] are distributed across [[GenericNumber]] dimensions: - GenericNumber: aggregation: default @@ -141,7 +141,7 @@ cards: score: 90 group: General - CountByCategoryMedium: - title: "[[this]] per [[GenericCategoryMedium]]" + title: "[[this.short-name]] per [[GenericCategoryMedium]]" dimensions: GenericCategoryMedium metrics: Count visualization: row @@ -151,7 +151,7 @@ cards: order_by: - Count: descending - CountByCategoryLarge: - title: "[[this]] per [[GenericCategoryLarge]]" + title: "[[this.short-name]] per [[GenericCategoryLarge]]" dimensions: GenericCategoryLarge metrics: Count visualization: table @@ -162,7 +162,7 @@ cards: - Count: descending # Geographical - CountByCountry: - title: "[[this]] per country" + title: "[[this.short-name]] per country" metrics: Count dimensions: Country score: 90 @@ -173,7 +173,7 @@ cards: group: Geographical height: 6 - CountByState: - title: "[[this]] per state" + title: "[[this.short-name]] per state" metrics: Count dimensions: State score: 90 @@ -184,7 +184,7 @@ cards: group: Geographical height: 6 - CountByCoords: - title: "[[this]] by coordinates" + title: "[[this.short-name]] by coordinates" metrics: Count dimensions: - Long @@ -195,42 +195,42 @@ cards: height: 6 # By Time - CountByJoinDate: - title: "[[this]] that have joined over time" + title: "[[this.short-name]] that have joined over time" visualization: line dimensions: JoinTimestamp metrics: Count score: 90 group: ByTime - CountByJoinDate: - title: "[[this]] that have joined over time" + title: "[[this.short-name]] that have joined over time" visualization: line dimensions: JoinDate metrics: Count score: 90 group: ByTime - CountByCreateDate: - title: New [[this]] over time + title: New [[this.short-name]] over time visualization: line dimensions: CreateTimestamp metrics: Count score: 90 group: ByTime - CountByCreateDate: - title: New [[this]] over time + title: New [[this.short-name]] over time visualization: line dimensions: CreateDate metrics: Count score: 90 group: ByTime - CountByTimestamp: - title: "[[this]] by [[Timestamp]]" + title: "[[this.short-name]] by [[Timestamp]]" visualization: line dimensions: Timestamp metrics: Count score: 20 group: ByTime - CountByTimestamp: - title: "[[this]] by [[Timestamp]]" + title: "[[this.short-name]] by [[Timestamp]]" visualization: line dimensions: Date metrics: Count @@ -371,7 +371,7 @@ cards: group: ByTime x_label: "[[Timestamp]]" - DayOfWeekCreateDate: - title: Weekdays when new [[this]] were added + title: Weekdays when new [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -381,7 +381,7 @@ cards: group: ByTime x_label: Created At by day of the week - DayOfWeekCreateDate: - title: Weekdays when new [[this]] were added + title: Weekdays when new [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -391,7 +391,7 @@ cards: group: ByTime x_label: Created At by day of the week - HourOfDayCreateDate: - title: Hours when new [[this]] were added + title: Hours when new [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -401,7 +401,7 @@ cards: group: ByTime x_label: Created At by hour of the day - HourOfDayCreateDate: - title: Hours when new [[this]] were added + title: Hours when new [[this.short-name]] were added visualization: bar dimensions: - CreateTime: @@ -411,7 +411,7 @@ cards: group: ByTime x_label: Created At by hour of the day - DayOfMonthCreateDate: - title: Days when new [[this]] were added + title: Days when new [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -421,7 +421,7 @@ cards: group: ByTime x_label: Created At by day of the month - DayOfMonthCreateDate: - title: Days when new [[this]] were added + title: Days when new [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -431,7 +431,7 @@ cards: group: ByTime x_label: Created At by day of the month - MonthOfYearCreateDate: - title: Months when new [[this]] were added + title: Months when new [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -441,7 +441,7 @@ cards: group: ByTime x_label: Created At by month of the year - MonthOfYearCreateDate: - title: Months when new [[this]] were added + title: Months when new [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -451,7 +451,7 @@ cards: group: ByTime x_label: Created At by month of the year - QuerterOfYearCreateDate: - title: Quarters when new [[this]] were added + title: Quarters when new [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -461,7 +461,7 @@ cards: group: ByTime x_label: Created At by quarter of the year - QuerterOfYearCreateDate: - title: Quarters when new [[this]] were added + title: Quarters when new [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -471,7 +471,7 @@ cards: group: ByTime x_label: Created At by quarter of the year - DayOfWeekJoinDate: - title: Weekdays when [[this]] joined + title: Weekdays when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -481,7 +481,7 @@ cards: group: ByTime x_label: Join date by day of the week - DayOfWeekJoinDate: - title: Weekdays when [[this]] joined + title: Weekdays when [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -491,7 +491,7 @@ cards: group: ByTime x_label: Join date by day of the week - HourOfDayJoinDate: - title: Hours when [[this]] joined + title: Hours when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -501,7 +501,7 @@ cards: group: ByTime x_label: Join date by hour of the day - HourOfDayJoinDate: - title: Hours when [[this]] joined + title: Hours when [[this.short-name]] joined visualization: bar dimensions: - JoinTime: @@ -511,7 +511,7 @@ cards: group: ByTime x_label: Join date by hour of the day - DayOfMonthJoinDate: - title: Days of the month when [[this]] joined + title: Days of the month when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -521,7 +521,7 @@ cards: group: ByTime x_label: Join date by day of the month - DayOfMonthJoinDate: - title: Days of the month when [[this]] joined + title: Days of the month when [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -531,7 +531,7 @@ cards: group: ByTime x_label: Join date by day of the month - MonthOfYearJoinDate: - title: Months when [[this]] joined + title: Months when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -541,7 +541,7 @@ cards: group: ByTime x_label: Join date by month of the year - MonthOfYearJoinDate: - title: Months when [[this]] joined + title: Months when [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -551,7 +551,7 @@ cards: group: ByTime x_label: Join date by month of the year - QuerterOfYearJoinDate: - title: Quarters when [[this]] joined + title: Quarters when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -561,7 +561,7 @@ cards: group: ByTime x_label: Join date by quarter of the year - QuerterOfYearJoinDate: - title: Quarters when [[this]] joined + title: Quarters when [[this.short-name]] joined visualization: bar dimensions: - JoinDate: diff --git a/resources/automagic_dashboards/table/UserTable.yaml b/resources/automagic_dashboards/table/UserTable.yaml index 1844a20527cb0..67cdc59cbb831 100644 --- a/resources/automagic_dashboards/table/UserTable.yaml +++ b/resources/automagic_dashboards/table/UserTable.yaml @@ -54,9 +54,9 @@ groups: - Overview: title: Overview - Geographical: - title: Where these [[this]] are + title: Where these [[this.short-title]] are - General: - title: How these [[this]] are distributed + title: How these [[this.short-title]] are distributed dashboard_filters: - JoinDate - GenericCategoryMedium @@ -66,7 +66,7 @@ dashboard_filters: cards: # Overview - Rowcount: - title: Total [[this]] + title: Total [[this.short-title]] visualization: scalar metrics: Count score: 100 @@ -74,7 +74,7 @@ cards: width: 5 height: 3 - RowcountLast30Days: - title: New [[this]] in the last 30 days + title: New [[this.short-title]] in the last 30 days visualization: scalar metrics: Count score: 100 @@ -84,8 +84,8 @@ cards: height: 3 - NewUsersByMonth: visualization: line - title: New [[this]] per month - description: The number of new [[this]] each month + title: New [[this.short-tiltle]] per month + description: The number of new [[this.short-title]] each month dimensions: JoinDate metrics: Count score: 100 @@ -94,7 +94,7 @@ cards: height: 7 # Geographical - CountByCountry: - title: Number of [[this]] per country + title: Number of [[this.short-title]] per country metrics: Count dimensions: Country score: 90 @@ -104,7 +104,7 @@ cards: map.region: world_countries group: Geographical - CountByState: - title: "[[this]] per state" + title: "[[this.short-title]] per state" metrics: Count dimensions: State score: 90 @@ -115,7 +115,7 @@ cards: map.region: us_states group: Geographical - CountByCoords: - title: "[[this]] by coordinates" + title: "[[this.short-title]] by coordinates" metrics: Count dimensions: - Long @@ -135,7 +135,7 @@ cards: score: 90 group: General - CountByCategoryMedium: - title: "[[this]] per [[GenericCategoryMedium]]" + title: "[[this.short-title]] per [[GenericCategoryMedium]]" dimensions: GenericCategoryMedium metrics: Count visualization: row @@ -145,7 +145,7 @@ cards: order_by: - Count: descending - CountByCategoryLarge: - title: "[[this]] per [[GenericCategoryLarge]]" + title: "[[this.short-title]] per [[GenericCategoryLarge]]" dimensions: GenericCategoryLarge metrics: Count visualization: table diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 7124de88ef169..d0089971d887c 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -116,6 +116,7 @@ :full-name (if (isa? (:entity_type table) :entity/GoogleAnalyticsTable) (:display_name table) (tru "{0} table" (:display_name table))) + :short-name (:display_name table) :source table :database (:db_id table) :url (format "%stable/%s" public-endpoint (u/get-id table)) @@ -126,6 +127,7 @@ (let [table (-> segment :table_id Table)] {:entity segment :full-name (tru "{0} in {1} segment" (:display_name table) (:name segment)) + :short-name (tru "such {0}" (:display_name table)) :source table :database (:db_id table) :query-filter (-> segment :definition :filter) @@ -137,6 +139,7 @@ (let [table (-> metric :table_id Table)] {:entity metric :full-name (tru "{0} metric" (:name metric)) + :short-name (:name metric) :source table :database (:db_id table) ;; We use :id here as it might not be a concrete field but rather one from a nested query which @@ -149,6 +152,7 @@ (let [table (field/table field)] {:entity field :full-name (tru "{0} field" (:display_name field)) + :short-name (:display_name field) :source table :database (:db_id table) ;; We use :id here as it might not be a concrete metric but rather one from a nested query @@ -353,8 +357,11 @@ bindings) (comp first #(filter-tables % tables) rules/->entity) identity)] - (str/replace s #"\[\[(\w+)\]\]" (fn [[_ identifier]] - (->reference template-type (bindings identifier)))))) + (str/replace s #"\[\[(\w+)(?:\.([\w\-]+))?\]\]" + (fn [[_ identifier attribute]] + (let [entity (bindings identifier)] + (or (some-> attribute qp.util/normalize-token root) + (->reference template-type entity))))))) (defn- field-candidates [context {:keys [field_type links_to named max_cardinality] :as constraints}] @@ -860,9 +867,10 @@ (-> dashboard :context :filters u/pprint-to-str)) (-> (cond-> dashboard cell-query - (assoc :title (tru "A closer look at {0}" - (cell-title (:context dashboard) cell-query)) - :transient_title nil) + (assoc :transient_title nil + :title (tru "A closer look at {0}" (cell-title (:context dashboard) + cell-query))) + query-filter (assoc :title (tru "A closer look at {0}" (:full-name root)))) (populate/create-dashboard (or show max-cards)) @@ -935,7 +943,7 @@ (u/get-id card) (encode-base64-json cell-query)) :entity (:source root) - :full-name (->> root + :short-name (->> root :source ((some-fn :display_name :name)) (tru "such {0}")) @@ -957,7 +965,7 @@ (encode-base64-json (:dataset_query query)) (encode-base64-json cell-query)) :entity (:source root) - :full-name (->> root + :short-name (->> root :source ((some-fn :display_name :name)) (tru "such {0}")) From 6773f44a1e8d529a93e2ec78efbfaeb7c485442b Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Fri, 22 Jun 2018 11:51:43 +0200 Subject: [PATCH 007/116] Alow acces to both root and what's binded to this via dot notation --- src/metabase/automagic_dashboards/core.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index d0089971d887c..fce5d77833f0f 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -359,8 +359,10 @@ identity)] (str/replace s #"\[\[(\w+)(?:\.([\w\-]+))?\]\]" (fn [[_ identifier attribute]] - (let [entity (bindings identifier)] - (or (some-> attribute qp.util/normalize-token root) + (let [entity (bindings identifier) + attribute (some-> attribute qp.util/normalize-token)] + (or (entity attribute) + (root attribute) (->reference template-type entity))))))) (defn- field-candidates From e7cabd936c33c04106105f14cb883652fd7530e2 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 25 Jun 2018 13:58:31 -0700 Subject: [PATCH 008/116] Implement nested collection item picker, use across app --- frontend/src/metabase-lib/lib/Question.js | 4 +- .../containers/CollectionCreate.jsx | 14 +- .../components/CreateDashboardModal.jsx | 43 ++-- frontend/src/metabase/components/Select.jsx | 22 +- .../src/metabase/components/SelectButton.jsx | 4 +- .../metabase/components/form/FormField.jsx | 5 +- .../metabase/components/form/FormWidget.jsx | 2 + .../metabase/components/form/StandardForm.jsx | 4 +- .../form/widgets/FormCollectionWidget.jsx | 7 + .../containers/AddToDashSelectDashModal.jsx | 72 ++----- .../containers/CollectionMoveModal.jsx | 34 ++- .../metabase/containers/CollectionName.jsx | 22 ++ .../metabase/containers/CollectionPicker.jsx | 122 ++--------- .../metabase/containers/CollectionSelect.jsx | 58 +---- .../src/metabase/containers/DashboardForm.jsx | 17 ++ .../metabase/containers/DashboardPicker.jsx | 22 ++ frontend/src/metabase/containers/Form.jsx | 10 +- frontend/src/metabase/containers/ItemName.jsx | 0 .../src/metabase/containers/ItemPicker.jsx | 202 ++++++++++++++++++ .../src/metabase/containers/ItemSelect.jsx | 60 ++++++ .../src/metabase/containers/QuestionName.jsx | 19 ++ .../metabase/containers/QuestionPicker.jsx | 22 ++ .../metabase/containers/QuestionSelect.jsx | 10 + .../AddToDashSelectQuestionModal.jsx | 13 +- frontend/src/metabase/entities/collections.js | 8 +- frontend/src/metabase/entities/pulses.js | 7 +- frontend/src/metabase/entities/questions.js | 22 +- frontend/src/metabase/lib/entities.js | 9 +- .../metabase/pulse/components/PulseEdit.jsx | 6 +- .../pulse/components/PulseEditCards.jsx | 12 +- .../pulse/components/PulseEditCollection.jsx | 10 +- .../qb/components/TimeseriesFilterWidget.jsx | 2 +- .../components/TimeseriesGroupingWidget.jsx | 2 +- .../query_builder/components/QueryHeader.jsx | 1 - .../questions/containers/AddToDashboard.jsx | 149 +------------ src/metabase/api/collection.clj | 2 +- src/metabase/models/collection.clj | 6 + 37 files changed, 550 insertions(+), 474 deletions(-) create mode 100644 frontend/src/metabase/components/form/widgets/FormCollectionWidget.jsx create mode 100644 frontend/src/metabase/containers/CollectionName.jsx create mode 100644 frontend/src/metabase/containers/DashboardForm.jsx create mode 100644 frontend/src/metabase/containers/DashboardPicker.jsx create mode 100644 frontend/src/metabase/containers/ItemName.jsx create mode 100644 frontend/src/metabase/containers/ItemPicker.jsx create mode 100644 frontend/src/metabase/containers/ItemSelect.jsx create mode 100644 frontend/src/metabase/containers/QuestionName.jsx create mode 100644 frontend/src/metabase/containers/QuestionPicker.jsx create mode 100644 frontend/src/metabase/containers/QuestionSelect.jsx diff --git a/frontend/src/metabase-lib/lib/Question.js b/frontend/src/metabase-lib/lib/Question.js index 95cb08682d3c6..59017e5cbbec2 100644 --- a/frontend/src/metabase-lib/lib/Question.js +++ b/frontend/src/metabase-lib/lib/Question.js @@ -491,7 +491,9 @@ export default class Question { } async reduxUpdate(dispatch) { - const { payload } = await dispatch(Questions.actions.update(this.card())); + const { payload } = await dispatch( + Questions.actions.update({ id: this.id() }, this.card()), + ); return this.setCard(payload.entities.questions[payload.result]); } diff --git a/frontend/src/metabase/collections/containers/CollectionCreate.jsx b/frontend/src/metabase/collections/containers/CollectionCreate.jsx index fb737d0c93b67..23234977a9d39 100644 --- a/frontend/src/metabase/collections/containers/CollectionCreate.jsx +++ b/frontend/src/metabase/collections/containers/CollectionCreate.jsx @@ -9,16 +9,14 @@ export default class CollectionCreate extends Component { render() { const { push, params } = this.props; const collectionId = - params && params.collectionId && parseFloat(params.collectionId); + params && params.collectionId != null && params.collectionId !== "root" + ? parseInt(params.collectionId) + : null; return ( push(`/collection/${id}`)} onClose={this.props.goBack} /> diff --git a/frontend/src/metabase/components/CreateDashboardModal.jsx b/frontend/src/metabase/components/CreateDashboardModal.jsx index 303ab0ac32de3..ccd913d4f43e8 100644 --- a/frontend/src/metabase/components/CreateDashboardModal.jsx +++ b/frontend/src/metabase/components/CreateDashboardModal.jsx @@ -12,27 +12,20 @@ import CollectionSelect from "metabase/containers/CollectionSelect.jsx"; import Dashboards from "metabase/entities/dashboards"; -const mapDispatchToProps = { - createDashboard: Dashboards.actions.create, -}; - -@connect(null, mapDispatchToProps) -@withRouter -export default class CreateDashboardModal extends Component { +export class CreateDashboardModal extends Component { constructor(props, context) { super(props, context); - this.createNewDash = this.createNewDash.bind(this); - this.setDescription = this.setDescription.bind(this); - this.setName = this.setName.bind(this); - - console.log(props.params); this.state = { name: null, description: null, errors: null, - // collectionId in the url starts off as a string, but the select will - // compare it to the integer ID on colleciton objects - collection_id: parseInt(props.params.collectionId), + collection_id: + props.collectionId != null + ? props.collectionId + : props.params.collectionId != null && + props.params.collectionId !== "root" + ? parseInt(props.params.collectionId) + : null, }; } @@ -41,15 +34,15 @@ export default class CreateDashboardModal extends Component { onClose: PropTypes.func, }; - setName(event) { + setName = event => { this.setState({ name: event.target.value }); - } + }; - setDescription(event) { + setDescription = event => { this.setState({ description: event.target.value }); - } + }; - createNewDash(event) { + createNewDash = event => { event.preventDefault(); let name = this.state.name && this.state.name.trim(); @@ -64,7 +57,7 @@ export default class CreateDashboardModal extends Component { this.props.createDashboard(newDash, { redirect: true }); this.props.onClose(); - } + }; render() { let formError; @@ -147,3 +140,11 @@ export default class CreateDashboardModal extends Component { ); } } + +const mapDispatchToProps = { + createDashboard: Dashboards.actions.create, +}; + +export default connect(null, mapDispatchToProps)( + withRouter(CreateDashboardModal), +); diff --git a/frontend/src/metabase/components/Select.jsx b/frontend/src/metabase/components/Select.jsx index 037164d5df09a..1fb20041ab7e9 100644 --- a/frontend/src/metabase/components/Select.jsx +++ b/frontend/src/metabase/components/Select.jsx @@ -8,6 +8,7 @@ import { t } from "c-3po"; import ColumnarSelector from "metabase/components/ColumnarSelector.jsx"; import Icon from "metabase/components/Icon.jsx"; import PopoverWithTrigger from "metabase/components/PopoverWithTrigger.jsx"; +import SelectButton from "./SelectButton"; import cx from "classnames"; import _ from "underscore"; @@ -207,27 +208,6 @@ class BrowserSelect extends Component { } } -export const SelectButton = ({ hasValue, children }) => ( -
- {children} - -
-); - -SelectButton.propTypes = { - hasValue: PropTypes.bool, - children: PropTypes.any, -}; - export class Option extends Component { static propTypes = { children: PropTypes.any, diff --git a/frontend/src/metabase/components/SelectButton.jsx b/frontend/src/metabase/components/SelectButton.jsx index 876bb74645242..1a17421adf339 100644 --- a/frontend/src/metabase/components/SelectButton.jsx +++ b/frontend/src/metabase/components/SelectButton.jsx @@ -6,8 +6,9 @@ import Icon from "metabase/components/Icon.jsx"; import cx from "classnames"; -const SelectButton = ({ className, children, hasValue = true }) => ( +const SelectButton = ({ className, style, children, hasValue = true }) => (
( SelectButton.propTypes = { className: PropTypes.string, + style: PropTypes.object, children: PropTypes.any, hasValue: PropTypes.any, }; diff --git a/frontend/src/metabase/components/form/FormField.jsx b/frontend/src/metabase/components/form/FormField.jsx index 12b130e8e016e..f966abbfac826 100644 --- a/frontend/src/metabase/components/form/FormField.jsx +++ b/frontend/src/metabase/components/form/FormField.jsx @@ -13,7 +13,10 @@ export default class FormField extends Component { active: PropTypes.bool, displayName: PropTypes.string, - children: PropTypes.element, + children: PropTypes.oneOfType([ + PropTypes.arrayOf(PropTypes.node), + PropTypes.node, + ]), // legacy fieldName: PropTypes.string, diff --git a/frontend/src/metabase/components/form/FormWidget.jsx b/frontend/src/metabase/components/form/FormWidget.jsx index 77e82001ca8f2..5fa968b4bfebe 100644 --- a/frontend/src/metabase/components/form/FormWidget.jsx +++ b/frontend/src/metabase/components/form/FormWidget.jsx @@ -5,6 +5,7 @@ import FormTextAreaWidget from "./widgets/FormTextAreaWidget"; import FormPasswordWidget from "./widgets/FormPasswordWidget"; import FormColorWidget from "./widgets/FormColorWidget"; import FormSelectWidget from "./widgets/FormSelectWidget"; +import FormCollectionWidget from "./widgets/FormCollectionWidget"; const WIDGETS = { input: FormInputWidget, @@ -12,6 +13,7 @@ const WIDGETS = { color: FormColorWidget, password: FormPasswordWidget, select: FormSelectWidget, + collection: FormCollectionWidget, }; const FormWidget = ({ type, ...props }) => { diff --git a/frontend/src/metabase/components/form/StandardForm.jsx b/frontend/src/metabase/components/form/StandardForm.jsx index dfc1c45431525..23bbfdb39793b 100644 --- a/frontend/src/metabase/components/form/StandardForm.jsx +++ b/frontend/src/metabase/components/form/StandardForm.jsx @@ -19,9 +19,9 @@ const StandardForm = ({ handleSubmit, resetForm, - form, + formDef: form, className, - resetButton = true, + resetButton = false, newForm = true, ...props diff --git a/frontend/src/metabase/components/form/widgets/FormCollectionWidget.jsx b/frontend/src/metabase/components/form/widgets/FormCollectionWidget.jsx new file mode 100644 index 0000000000000..cd66d10e87652 --- /dev/null +++ b/frontend/src/metabase/components/form/widgets/FormCollectionWidget.jsx @@ -0,0 +1,7 @@ +import React from "react"; + +import CollectionSelect from "metabase/containers/CollectionSelect"; + +const FormCollectionWidget = ({ field }) => ; + +export default FormCollectionWidget; diff --git a/frontend/src/metabase/containers/AddToDashSelectDashModal.jsx b/frontend/src/metabase/containers/AddToDashSelectDashModal.jsx index 4fd911835d4c7..e23bc1432269a 100644 --- a/frontend/src/metabase/containers/AddToDashSelectDashModal.jsx +++ b/frontend/src/metabase/containers/AddToDashSelectDashModal.jsx @@ -2,29 +2,17 @@ import React, { Component } from "react"; import { connect } from "react-redux"; +import { t } from "c-3po"; -import CreateDashboardModal from "metabase/components/CreateDashboardModal.jsx"; -import Icon from "metabase/components/Icon.jsx"; import ModalContent from "metabase/components/ModalContent.jsx"; -import SortableItemList from "metabase/components/SortableItemList.jsx"; -import * as Urls from "metabase/lib/urls"; +import DashboardForm from "metabase/containers/DashboardForm.jsx"; +import DashboardPicker from "metabase/containers/DashboardPicker"; -import Dashboards from "metabase/entities/dashboards"; +import * as Urls from "metabase/lib/urls"; -import { t } from "c-3po"; import type { Dashboard, DashboardId } from "metabase/meta/types/Dashboard"; import type { Card } from "metabase/meta/types/Card"; -const mapStateToProps = state => ({ - dashboards: Dashboards.selectors.getList(state), -}); - -const mapDispatchToProps = { - fetchDashboards: Dashboards.actions.fetchList, - createDashboard: Dashboards.actions.create, -}; - -@connect(mapStateToProps, mapDispatchToProps) export default class AddToDashSelectDashModal extends Component { state = { shouldCreateDashboard: false, @@ -35,15 +23,9 @@ export default class AddToDashSelectDashModal extends Component { onClose: () => void, onChangeLocation: string => void, // via connect: - dashboards: Dashboard[], - fetchDashboards: () => any, createDashboard: Dashboard => any, }; - componentWillMount() { - this.props.fetchDashboards(); - } - addToDashboard = (dashboardId: DashboardId) => { // we send the user over to the chosen dashboard in edit mode with the current card added this.props.onChangeLocation( @@ -51,26 +33,12 @@ export default class AddToDashSelectDashModal extends Component { ); }; - createDashboard = async (newDashboard: Dashboard) => { - try { - const action = await this.props.createDashboard(newDashboard); - this.addToDashboard(action.payload.result); - } catch (e) { - console.log("createDashboard failed", e); - } - }; - render() { - if (this.props.dashboards === null) { - return
; - } else if ( - this.props.dashboards.length === 0 || - this.state.shouldCreateDashboard === true - ) { + if (this.state.shouldCreateDashboard) { return ( - this.addToDashboard(dashboard.id)} /> ); } else { @@ -80,24 +48,12 @@ export default class AddToDashSelectDashModal extends Component { title={t`Add Question to Dashboard`} onClose={this.props.onClose} > -
-
this.setState({ shouldCreateDashboard: true })} - > -
- -

{t`Add to new dashboard`}

-
-
- this.addToDashboard(dashboard.id)} - /> -
+ + ); } diff --git a/frontend/src/metabase/containers/CollectionMoveModal.jsx b/frontend/src/metabase/containers/CollectionMoveModal.jsx index 2ee3045f31e94..bcadcd2cca5bd 100644 --- a/frontend/src/metabase/containers/CollectionMoveModal.jsx +++ b/frontend/src/metabase/containers/CollectionMoveModal.jsx @@ -24,10 +24,7 @@ class CollectionMoveModal extends React.Component { // null = root collection // number = non-root collection id // - selectedCollection: - props.initialCollectionId === undefined - ? undefined - : { id: props.initialCollectionId }, + selectedCollectionId: props.initialCollectionId, // whether the move action has started // TODO: use this loading and error state in the UI moving: false, @@ -43,7 +40,7 @@ class CollectionMoveModal extends React.Component { }; render() { - const { selectedCollection } = this.state; + const { selectedCollectionId } = this.state; return ( @@ -55,29 +52,24 @@ class CollectionMoveModal extends React.Component { onClick={() => this.props.onClose()} /> - - {({ collections, loading, error }) => ( - - this.setState({ - selectedCollection: - id == null ? null : _.find(collections, { id }), - }) - } - collections={collections} - /> - )} - + + this.setState({ selectedCollectionId }) + } + />
diff --git a/frontend/src/metabase/pulse/components/PulseEditCollection.jsx b/frontend/src/metabase/pulse/components/PulseEditCollection.jsx index 9d6a16b9e3a9c..461801aa9dbec 100644 --- a/frontend/src/metabase/pulse/components/PulseEditCollection.jsx +++ b/frontend/src/metabase/pulse/components/PulseEditCollection.jsx @@ -6,6 +6,7 @@ import CollectionSelect from "metabase/containers/CollectionSelect"; export default class PulseEditCollection extends React.Component { render() { + const { pulse, setPulse, initialCollectionId } = this.props; return (

{t`Which collection should this pulse live in?`}

@@ -13,12 +14,13 @@ export default class PulseEditCollection extends React.Component { - this.props.setPulse({ - ...this.props.pulse, + setPulse({ + ...pulse, collection_id, }) } diff --git a/frontend/src/metabase/qb/components/TimeseriesFilterWidget.jsx b/frontend/src/metabase/qb/components/TimeseriesFilterWidget.jsx index 5fab16f1332c8..f6cce5c5637b7 100644 --- a/frontend/src/metabase/qb/components/TimeseriesFilterWidget.jsx +++ b/frontend/src/metabase/qb/components/TimeseriesFilterWidget.jsx @@ -4,7 +4,7 @@ import React, { Component } from "react"; import { t } from "c-3po"; import DatePicker from "metabase/query_builder/components/filters/pickers/DatePicker"; import PopoverWithTrigger from "metabase/components/PopoverWithTrigger"; -import { SelectButton } from "metabase/components/Select"; +import SelectButton from "metabase/components/SelectButton"; import Button from "metabase/components/Button"; import * as Query from "metabase/lib/query/query"; diff --git a/frontend/src/metabase/qb/components/TimeseriesGroupingWidget.jsx b/frontend/src/metabase/qb/components/TimeseriesGroupingWidget.jsx index 457bd7584d53d..7ffee1492c4c9 100644 --- a/frontend/src/metabase/qb/components/TimeseriesGroupingWidget.jsx +++ b/frontend/src/metabase/qb/components/TimeseriesGroupingWidget.jsx @@ -4,7 +4,7 @@ import React, { Component } from "react"; import TimeGroupingPopover from "metabase/query_builder/components/TimeGroupingPopover"; import PopoverWithTrigger from "metabase/components/PopoverWithTrigger"; -import { SelectButton } from "metabase/components/Select"; +import SelectButton from "metabase/components/SelectButton"; import * as Query from "metabase/lib/query/query"; import * as Card from "metabase/meta/Card"; diff --git a/frontend/src/metabase/query_builder/components/QueryHeader.jsx b/frontend/src/metabase/query_builder/components/QueryHeader.jsx index 0127fc7201886..a603e657106a2 100644 --- a/frontend/src/metabase/query_builder/components/QueryHeader.jsx +++ b/frontend/src/metabase/query_builder/components/QueryHeader.jsx @@ -281,7 +281,6 @@ export default class QueryHeader extends Component { } onClose={onClose} onMove={collection => { - this.props.onSetCardAttribute("collection", collection); this.props.onSetCardAttribute( "collection_id", collection && collection.id, diff --git a/frontend/src/metabase/questions/containers/AddToDashboard.jsx b/frontend/src/metabase/questions/containers/AddToDashboard.jsx index 6dd15e6cacfaf..b10018efa16e9 100644 --- a/frontend/src/metabase/questions/containers/AddToDashboard.jsx +++ b/frontend/src/metabase/questions/containers/AddToDashboard.jsx @@ -1,151 +1,18 @@ import React, { Component } from "react"; import { t } from "c-3po"; -import ModalContent from "metabase/components/ModalContent.jsx"; -import Icon from "metabase/components/Icon.jsx"; -import HeaderWithBack from "metabase/components/HeaderWithBack"; -import QuestionIcon from "metabase/components/QuestionIcon"; - -import CollectionListLoader from "metabase/containers/CollectionListLoader"; -import QuestionListLoader from "metabase/containers/QuestionListLoader"; - -import ExpandingSearchField from "../components/ExpandingSearchField.jsx"; -const QuestionRow = ({ question, onClick }) => ( -
-
- -
-
- {question.name} -
- {question.description ? ( -
{question.description}
- ) : ( -
{`No description yet`}
- )} -
-
-
-); +import ModalContent from "metabase/components/ModalContent.jsx"; +import QuestionPicker from "metabase/containers/QuestionPicker"; export default class AddToDashboard extends Component { - state = { - collection: null, - query: null, - }; - - renderQuestionList = () => { - return ( - - {({ questions }) => ( -
- {questions.map(question => ( - this.props.onAdd(question)} - /> - ))} -
- )} - - ); - }; - - renderCollections = () => { - return ( - - {({ collections }) => ( -
- {/* only show the collections list if there are actually collections fixes #4668 */ - collections.length > 0 ? ( -
    - {collections.map((collection, index) => ( -
  1. - this.setState({ - collection: collection, - query: { collection: collection.slug }, - }) - } - > - -

    {collection.name}

    - -
    2. - ))} -
  2. - this.setState({ - collection: { name: t`Everything else` }, - query: { collection: "" }, - }) - } - > - -

    Everything else

    - -
    3. -
- ) : ( - this.renderQuestionList() - )} -
- )} - - ); - }; - render() { - const { query, collection } = this.state; return ( -
- this.props.onClose()} - > -
-
- {!query ? ( - - this.setState({ - collection: null, - query: { q: value }, - }) - } - /> - ) : ( - - this.setState({ collection: null, query: null }) - } - /> - )} -
-
-
- {query - ? // a search term has been entered so show the questions list - this.renderQuestionList() - : // show the collections list - this.renderCollections()} -
- -
+ + + ); } } diff --git a/src/metabase/api/collection.clj b/src/metabase/api/collection.clj index 4493b5cd518ca..4e5103eb70976 100644 --- a/src/metabase/api/collection.clj +++ b/src/metabase/api/collection.clj @@ -95,7 +95,7 @@ Works for either a normal Collection or the Root Collection." [collection :- collection/CollectionWithLocationAndIDOrRoot] (-> collection - (hydrate :effective_location :effective_ancestors :can_write))) + (hydrate :parent_id :effective_location :effective_ancestors :can_write))) (s/defn ^:private collection-items "Return items in the Collection, restricted by `children-options`. diff --git a/src/metabase/models/collection.clj b/src/metabase/models/collection.clj index d4f883758bcd3..09d5a673c38b3 100644 --- a/src/metabase/models/collection.clj +++ b/src/metabase/models/collection.clj @@ -296,6 +296,12 @@ [] (filter i/can-read? (ancestors collection)))) +(s/defn parent-id :- (s/maybe su/IntGreaterThanZero) + "Get the immediate parent `collection` id, if set." + {:hydrate :parent_id} + [{:keys [location]} :- CollectionWithLocationOrRoot] + (if location (location-path->parent-id location))) + (s/defn children-location :- LocationPath "Given a `collection` return a location path that should match the `:location` value of all the children of the Collection. From 2104b5a403276a849f5aabb92ac59a78d349c03c Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Mon, 25 Jun 2018 23:01:02 +0200 Subject: [PATCH 009/116] Don't stuff filter clauses into `cell-query`. --- src/metabase/automagic_dashboards/core.clj | 7 +++---- test/metabase/automagic_dashboards/core_test.clj | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index fce5d77833f0f..366d31ba1dd2c 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -361,7 +361,7 @@ (fn [[_ identifier attribute]] (let [entity (bindings identifier) attribute (some-> attribute qp.util/normalize-token)] - (or (entity attribute) + (or (and (ifn? entity) (entity attribute)) (root attribute) (->reference template-type entity))))))) @@ -967,14 +967,13 @@ (encode-base64-json (:dataset_query query)) (encode-base64-json cell-query)) :entity (:source root) + :query-filter (qp.util/get-in-normalized query [:dataset_query :query :filter]) :short-name (->> root :source ((some-fn :display_name :name)) (tru "such {0}")) :rules-prefix ["table"]})) - (update opts :cell-query - (partial filters/inject-refinement - (qp.util/get-in-normalized query [:dataset_query :query :filter]))))) + opts)) (let [opts (assoc opts :show :all)] (->> (decompose-question root query opts) (apply populate/merge-dashboards (automagic-dashboard root)) diff --git a/test/metabase/automagic_dashboards/core_test.clj b/test/metabase/automagic_dashboards/core_test.clj index fd17104868250..88fa19a03c700 100644 --- a/test/metabase/automagic_dashboards/core_test.clj +++ b/test/metabase/automagic_dashboards/core_test.clj @@ -217,7 +217,7 @@ (with-dashboard-cleanup (-> card-id Card - (automagic-analysis {:cell-query [:!= [:field-id (data/id :venues :category_id)] 2]}) + (automagic-analysis {:cell-query [:= [:field-id (data/id :venues :category_id)] 2]}) valid-dashboard?))))) From 106241aa3406913936229473939eed35b9d94510 Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Mon, 25 Jun 2018 17:12:39 -0700 Subject: [PATCH 010/116] small tweaks and fixes --- .../automagic_dashboards/field/Country.yaml | 2 +- .../automagic_dashboards/field/DateTime.yaml | 2 +- .../metric/GenericMetric.yaml | 6 ++--- .../table/GenericTable.yaml | 24 +++++++++---------- .../automagic_dashboards/table/UserTable.yaml | 2 +- src/metabase/automagic_dashboards/core.clj | 6 ++--- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/resources/automagic_dashboards/field/Country.yaml b/resources/automagic_dashboards/field/Country.yaml index 729c0d4e287cb..9cd91ba6b52cd 100644 --- a/resources/automagic_dashboards/field/Country.yaml +++ b/resources/automagic_dashboards/field/Country.yaml @@ -60,7 +60,7 @@ cards: group: Overview width: 6 - Distribution: - title: Distribution of [[this]] + title: How the [[this]] is distributed visualization: map: map.type: region diff --git a/resources/automagic_dashboards/field/DateTime.yaml b/resources/automagic_dashboards/field/DateTime.yaml index 38ccb0632cf2f..346f802e8882b 100644 --- a/resources/automagic_dashboards/field/DateTime.yaml +++ b/resources/automagic_dashboards/field/DateTime.yaml @@ -34,7 +34,7 @@ groups: - Breakdowns: title: How the [[this]] is distributed - Seasonality: - title: Seasonal patterns in [[this]] + title: Seasonal patterns in the [[this]] cards: - Count: title: Count diff --git a/resources/automagic_dashboards/metric/GenericMetric.yaml b/resources/automagic_dashboards/metric/GenericMetric.yaml index 3c8c22b092e45..6af5a040a0d81 100644 --- a/resources/automagic_dashboards/metric/GenericMetric.yaml +++ b/resources/automagic_dashboards/metric/GenericMetric.yaml @@ -35,15 +35,15 @@ dimensions: field_type: GenericTable.ZipCode groups: - Periodicity: - title: "[[this]] over time" + title: The [[this]] over time - Geographical: - title: "[[this]] by location" + title: The [[this]] by location - Categories: title: How the [[this]] is distributed across different categories - Numbers: title: How the [[this]] is distributed across different numbers - LargeCategories: - title: Top and bottom [[this]] + title: The top and bottom for the [[this]] dashboard_filters: - Timestamp - State diff --git a/resources/automagic_dashboards/table/GenericTable.yaml b/resources/automagic_dashboards/table/GenericTable.yaml index b7dbae520b086..f7e0bea3d985c 100644 --- a/resources/automagic_dashboards/table/GenericTable.yaml +++ b/resources/automagic_dashboards/table/GenericTable.yaml @@ -112,13 +112,13 @@ dashboard_filters: cards: # Overview - Rowcount: - title: Total [[this.short-name]] + title: The number of [[this.short-name]] visualization: scalar metrics: Count score: 100 group: Overview - RowcountLast30Days: - title: New [[this.short-name]] in the last 30 days + title: "[[this.short-name]] added in the last 30 days" visualization: scalar metrics: Count score: 100 @@ -371,7 +371,7 @@ cards: group: ByTime x_label: "[[Timestamp]]" - DayOfWeekCreateDate: - title: Weekdays when new [[this.short-name]] were added + title: Weekdays when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -381,7 +381,7 @@ cards: group: ByTime x_label: Created At by day of the week - DayOfWeekCreateDate: - title: Weekdays when new [[this.short-name]] were added + title: Weekdays when [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -391,7 +391,7 @@ cards: group: ByTime x_label: Created At by day of the week - HourOfDayCreateDate: - title: Hours when new [[this.short-name]] were added + title: Hours when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -401,7 +401,7 @@ cards: group: ByTime x_label: Created At by hour of the day - HourOfDayCreateDate: - title: Hours when new [[this.short-name]] were added + title: Hours when [[this.short-name]] were added visualization: bar dimensions: - CreateTime: @@ -411,7 +411,7 @@ cards: group: ByTime x_label: Created At by hour of the day - DayOfMonthCreateDate: - title: Days when new [[this.short-name]] were added + title: Days when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -421,7 +421,7 @@ cards: group: ByTime x_label: Created At by day of the month - DayOfMonthCreateDate: - title: Days when new [[this.short-name]] were added + title: Days when [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -431,7 +431,7 @@ cards: group: ByTime x_label: Created At by day of the month - MonthOfYearCreateDate: - title: Months when new [[this.short-name]] were added + title: Months when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -441,7 +441,7 @@ cards: group: ByTime x_label: Created At by month of the year - MonthOfYearCreateDate: - title: Months when new [[this.short-name]] were added + title: Months when [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -451,7 +451,7 @@ cards: group: ByTime x_label: Created At by month of the year - QuerterOfYearCreateDate: - title: Quarters when new [[this.short-name]] were added + title: Quarters when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -461,7 +461,7 @@ cards: group: ByTime x_label: Created At by quarter of the year - QuerterOfYearCreateDate: - title: Quarters when new [[this.short-name]] were added + title: Quarters when [[this.short-name]] were added visualization: bar dimensions: - CreateDate: diff --git a/resources/automagic_dashboards/table/UserTable.yaml b/resources/automagic_dashboards/table/UserTable.yaml index 67cdc59cbb831..3a47149330ce5 100644 --- a/resources/automagic_dashboards/table/UserTable.yaml +++ b/resources/automagic_dashboards/table/UserTable.yaml @@ -84,7 +84,7 @@ cards: height: 3 - NewUsersByMonth: visualization: line - title: New [[this.short-tiltle]] per month + title: New [[this.short-title]] per month description: The number of new [[this.short-title]] each month dimensions: JoinDate metrics: Count diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 366d31ba1dd2c..d618f74e4cbac 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -127,7 +127,7 @@ (let [table (-> segment :table_id Table)] {:entity segment :full-name (tru "{0} in {1} segment" (:display_name table) (:name segment)) - :short-name (tru "such {0}" (:display_name table)) + :short-name (tru "these {0}" (:display_name table)) :source table :database (:db_id table) :query-filter (-> segment :definition :filter) @@ -948,7 +948,7 @@ :short-name (->> root :source ((some-fn :display_name :name)) - (tru "such {0}")) + (tru "these {0}")) :rules-prefix ["table"]})) opts)) (let [opts (assoc opts :show :all)] @@ -971,7 +971,7 @@ :short-name (->> root :source ((some-fn :display_name :name)) - (tru "such {0}")) + (tru "these {0}")) :rules-prefix ["table"]})) opts)) (let [opts (assoc opts :show :all)] From 992ff4dba6eb0efa9adfda2193aaef5a17e381d1 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Tue, 26 Jun 2018 13:52:10 +0200 Subject: [PATCH 011/116] fix typo short-title -> short-name --- .../automagic_dashboards/table/UserTable.yaml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/resources/automagic_dashboards/table/UserTable.yaml b/resources/automagic_dashboards/table/UserTable.yaml index 3a47149330ce5..ebdd11cf07a62 100644 --- a/resources/automagic_dashboards/table/UserTable.yaml +++ b/resources/automagic_dashboards/table/UserTable.yaml @@ -54,9 +54,9 @@ groups: - Overview: title: Overview - Geographical: - title: Where these [[this.short-title]] are + title: Where these [[this.short-name]] are - General: - title: How these [[this.short-title]] are distributed + title: How these [[this.short-name]] are distributed dashboard_filters: - JoinDate - GenericCategoryMedium @@ -66,7 +66,7 @@ dashboard_filters: cards: # Overview - Rowcount: - title: Total [[this.short-title]] + title: Total [[this.short-name]] visualization: scalar metrics: Count score: 100 @@ -74,7 +74,7 @@ cards: width: 5 height: 3 - RowcountLast30Days: - title: New [[this.short-title]] in the last 30 days + title: New [[this.short-name]] in the last 30 days visualization: scalar metrics: Count score: 100 @@ -84,8 +84,8 @@ cards: height: 3 - NewUsersByMonth: visualization: line - title: New [[this.short-title]] per month - description: The number of new [[this.short-title]] each month + title: New [[this.short-name]] per month + description: The number of new [[this.short-name]] each month dimensions: JoinDate metrics: Count score: 100 @@ -94,7 +94,7 @@ cards: height: 7 # Geographical - CountByCountry: - title: Number of [[this.short-title]] per country + title: Number of [[this.short-name]] per country metrics: Count dimensions: Country score: 90 @@ -104,7 +104,7 @@ cards: map.region: world_countries group: Geographical - CountByState: - title: "[[this.short-title]] per state" + title: "[[this.short-name]] per state" metrics: Count dimensions: State score: 90 @@ -115,7 +115,7 @@ cards: map.region: us_states group: Geographical - CountByCoords: - title: "[[this.short-title]] by coordinates" + title: "[[this.short-name]] by coordinates" metrics: Count dimensions: - Long @@ -135,7 +135,7 @@ cards: score: 90 group: General - CountByCategoryMedium: - title: "[[this.short-title]] per [[GenericCategoryMedium]]" + title: "[[this.short-name]] per [[GenericCategoryMedium]]" dimensions: GenericCategoryMedium metrics: Count visualization: row @@ -145,7 +145,7 @@ cards: order_by: - Count: descending - CountByCategoryLarge: - title: "[[this.short-title]] per [[GenericCategoryLarge]]" + title: "[[this.short-name]] per [[GenericCategoryLarge]]" dimensions: GenericCategoryLarge metrics: Count visualization: table From a3abcaaf9d71f56c3cf826f67401f4cbdf7b3544 Mon Sep 17 00:00:00 2001 From: Kyle Doherty Date: Tue, 26 Jun 2018 11:38:26 -0400 Subject: [PATCH 012/116] cursor pointer on crumbs --- frontend/src/metabase/components/Breadcrumbs.css | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/metabase/components/Breadcrumbs.css b/frontend/src/metabase/components/Breadcrumbs.css index 586d2d9ff1ac0..1ec5e0de6d4f7 100644 --- a/frontend/src/metabase/components/Breadcrumbs.css +++ b/frontend/src/metabase/components/Breadcrumbs.css @@ -17,6 +17,7 @@ font-size: 0.75rem; font-weight: bold; text-transform: uppercase; + cursor: pointer; } :local(.breadcrumbDivider) { From d4bb6d36f3c00f85724a389e76c844b619afc29f Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Tue, 26 Jun 2018 18:38:09 +0200 Subject: [PATCH 013/116] Change cell xray to be based off card xray --- .../field/GenericField.yaml | 3 +- .../table/GenericTable.yaml | 66 ++-- .../automagic_dashboards/table/UserTable.yaml | 24 +- src/metabase/automagic_dashboards/core.clj | 296 +++++++++--------- 4 files changed, 196 insertions(+), 193 deletions(-) diff --git a/resources/automagic_dashboards/field/GenericField.yaml b/resources/automagic_dashboards/field/GenericField.yaml index a5ec4274d7dc9..fe5396453e26c 100644 --- a/resources/automagic_dashboards/field/GenericField.yaml +++ b/resources/automagic_dashboards/field/GenericField.yaml @@ -68,12 +68,13 @@ cards: width: 12 - ByNumber: title: "[[GenericNumber]] by [[this]]" - visualization: line + visualization: bar metrics: - Sum - Avg dimensions: this group: Breakdowns + height: 8 - Crosstab: title: "[[this]] by [[GenericCategoryMedium]]" visualization: table diff --git a/resources/automagic_dashboards/table/GenericTable.yaml b/resources/automagic_dashboards/table/GenericTable.yaml index f7e0bea3d985c..60f04bee42417 100644 --- a/resources/automagic_dashboards/table/GenericTable.yaml +++ b/resources/automagic_dashboards/table/GenericTable.yaml @@ -93,11 +93,11 @@ groups: - Singletons: title: These are the same for all your [[this.short-name]] - ByTime: - title: "[[this.short-name]] across time" + title: "These [[this.short-name]] across time" - Geographical: title: Where your [[this.short-name]] are - General: - title: How [[this.short-name]] are distributed + title: How these [[this.short-name]] are distributed dashboard_filters: - Timestamp - Date @@ -132,7 +132,7 @@ cards: group: Overview # General - NumberDistribution: - title: How [[this.short-name]] are distributed across [[GenericNumber]] + title: How these [[this.short-name]] are distributed across [[GenericNumber]] dimensions: - GenericNumber: aggregation: default @@ -141,7 +141,7 @@ cards: score: 90 group: General - CountByCategoryMedium: - title: "[[this.short-name]] per [[GenericCategoryMedium]]" + title: "These [[this.short-name]] per [[GenericCategoryMedium]]" dimensions: GenericCategoryMedium metrics: Count visualization: row @@ -151,7 +151,7 @@ cards: order_by: - Count: descending - CountByCategoryLarge: - title: "[[this.short-name]] per [[GenericCategoryLarge]]" + title: "These [[this.short-name]] per [[GenericCategoryLarge]]" dimensions: GenericCategoryLarge metrics: Count visualization: table @@ -162,7 +162,7 @@ cards: - Count: descending # Geographical - CountByCountry: - title: "[[this.short-name]] per country" + title: "These [[this.short-name]] per country" metrics: Count dimensions: Country score: 90 @@ -173,7 +173,7 @@ cards: group: Geographical height: 6 - CountByState: - title: "[[this.short-name]] per state" + title: "These [[this.short-name]] per state" metrics: Count dimensions: State score: 90 @@ -184,7 +184,7 @@ cards: group: Geographical height: 6 - CountByCoords: - title: "[[this.short-name]] by coordinates" + title: "These [[this.short-name]] by coordinates" metrics: Count dimensions: - Long @@ -195,14 +195,14 @@ cards: height: 6 # By Time - CountByJoinDate: - title: "[[this.short-name]] that have joined over time" + title: "These [[this.short-name]] that have joined over time" visualization: line dimensions: JoinTimestamp metrics: Count score: 90 group: ByTime - CountByJoinDate: - title: "[[this.short-name]] that have joined over time" + title: "These [[this.short-name]] that have joined over time" visualization: line dimensions: JoinDate metrics: Count @@ -223,21 +223,21 @@ cards: score: 90 group: ByTime - CountByTimestamp: - title: "[[this.short-name]] by [[Timestamp]]" + title: "These [[this.short-name]] by [[Timestamp]]" visualization: line dimensions: Timestamp metrics: Count score: 20 group: ByTime - CountByTimestamp: - title: "[[this.short-name]] by [[Timestamp]]" + title: "These [[this.short-name]] by [[Timestamp]]" visualization: line dimensions: Date metrics: Count score: 20 group: ByTime - NumberOverTime: - title: "[[GenericNumber]] over time" + title: "These [[GenericNumber]] over time" visualization: line dimensions: Timestamp metrics: @@ -371,7 +371,7 @@ cards: group: ByTime x_label: "[[Timestamp]]" - DayOfWeekCreateDate: - title: Weekdays when [[this.short-name]] were added + title: Weekdays when these [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -381,7 +381,7 @@ cards: group: ByTime x_label: Created At by day of the week - DayOfWeekCreateDate: - title: Weekdays when [[this.short-name]] were added + title: Weekdays when these [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -391,7 +391,7 @@ cards: group: ByTime x_label: Created At by day of the week - HourOfDayCreateDate: - title: Hours when [[this.short-name]] were added + title: Hours when these [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -401,7 +401,7 @@ cards: group: ByTime x_label: Created At by hour of the day - HourOfDayCreateDate: - title: Hours when [[this.short-name]] were added + title: Hours when these [[this.short-name]] were added visualization: bar dimensions: - CreateTime: @@ -411,7 +411,7 @@ cards: group: ByTime x_label: Created At by hour of the day - DayOfMonthCreateDate: - title: Days when [[this.short-name]] were added + title: Days when these [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -421,7 +421,7 @@ cards: group: ByTime x_label: Created At by day of the month - DayOfMonthCreateDate: - title: Days when [[this.short-name]] were added + title: Days when these [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -431,7 +431,7 @@ cards: group: ByTime x_label: Created At by day of the month - MonthOfYearCreateDate: - title: Months when [[this.short-name]] were added + title: Months when these [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -441,7 +441,7 @@ cards: group: ByTime x_label: Created At by month of the year - MonthOfYearCreateDate: - title: Months when [[this.short-name]] were added + title: Months when these [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -451,7 +451,7 @@ cards: group: ByTime x_label: Created At by month of the year - QuerterOfYearCreateDate: - title: Quarters when [[this.short-name]] were added + title: Quarters when these [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -461,7 +461,7 @@ cards: group: ByTime x_label: Created At by quarter of the year - QuerterOfYearCreateDate: - title: Quarters when [[this.short-name]] were added + title: Quarters when these [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -471,7 +471,7 @@ cards: group: ByTime x_label: Created At by quarter of the year - DayOfWeekJoinDate: - title: Weekdays when [[this.short-name]] joined + title: Weekdays when these [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -481,7 +481,7 @@ cards: group: ByTime x_label: Join date by day of the week - DayOfWeekJoinDate: - title: Weekdays when [[this.short-name]] joined + title: Weekdays when these [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -491,7 +491,7 @@ cards: group: ByTime x_label: Join date by day of the week - HourOfDayJoinDate: - title: Hours when [[this.short-name]] joined + title: Hours when these [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -501,7 +501,7 @@ cards: group: ByTime x_label: Join date by hour of the day - HourOfDayJoinDate: - title: Hours when [[this.short-name]] joined + title: Hours when these [[this.short-name]] joined visualization: bar dimensions: - JoinTime: @@ -511,7 +511,7 @@ cards: group: ByTime x_label: Join date by hour of the day - DayOfMonthJoinDate: - title: Days of the month when [[this.short-name]] joined + title: Days of the month when these [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -521,7 +521,7 @@ cards: group: ByTime x_label: Join date by day of the month - DayOfMonthJoinDate: - title: Days of the month when [[this.short-name]] joined + title: Days of the month when these [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -531,7 +531,7 @@ cards: group: ByTime x_label: Join date by day of the month - MonthOfYearJoinDate: - title: Months when [[this.short-name]] joined + title: Months when these [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -541,7 +541,7 @@ cards: group: ByTime x_label: Join date by month of the year - MonthOfYearJoinDate: - title: Months when [[this.short-name]] joined + title: Months when these [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -551,7 +551,7 @@ cards: group: ByTime x_label: Join date by month of the year - QuerterOfYearJoinDate: - title: Quarters when [[this.short-name]] joined + title: Quarters when these [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -561,7 +561,7 @@ cards: group: ByTime x_label: Join date by quarter of the year - QuerterOfYearJoinDate: - title: Quarters when [[this.short-name]] joined + title: Quarters when these [[this.short-name]] joined visualization: bar dimensions: - JoinDate: diff --git a/resources/automagic_dashboards/table/UserTable.yaml b/resources/automagic_dashboards/table/UserTable.yaml index ebdd11cf07a62..b6297d02f8361 100644 --- a/resources/automagic_dashboards/table/UserTable.yaml +++ b/resources/automagic_dashboards/table/UserTable.yaml @@ -23,14 +23,11 @@ dimensions: - JoinDate: field_type: Date score: 30 -- Source: - field_type: Source - GenericNumber: field_type: GenericTable.Number score: 80 - Source: field_type: GenericTable.Source - score: 100 - GenericCategoryMedium: field_type: GenericTable.Category score: 75 @@ -85,7 +82,6 @@ cards: - NewUsersByMonth: visualization: line title: New [[this.short-name]] per month - description: The number of new [[this.short-name]] each month dimensions: JoinDate metrics: Count score: 100 @@ -94,7 +90,7 @@ cards: height: 7 # Geographical - CountByCountry: - title: Number of [[this.short-name]] per country + title: Per country metrics: Count dimensions: Country score: 90 @@ -104,7 +100,7 @@ cards: map.region: world_countries group: Geographical - CountByState: - title: "[[this.short-name]] per state" + title: "Per state" metrics: Count dimensions: State score: 90 @@ -115,7 +111,7 @@ cards: map.region: us_states group: Geographical - CountByCoords: - title: "[[this.short-name]] by coordinates" + title: "By coordinates" metrics: Count dimensions: - Long @@ -135,7 +131,7 @@ cards: score: 90 group: General - CountByCategoryMedium: - title: "[[this.short-name]] per [[GenericCategoryMedium]]" + title: "Per [[GenericCategoryMedium]]" dimensions: GenericCategoryMedium metrics: Count visualization: row @@ -144,8 +140,18 @@ cards: group: General order_by: - Count: descending + - CountBySource: + title: "Per [[Source]]" + dimensions: Source + metrics: Count + visualization: row + score: 80 + height: 8 + group: General + order_by: + - Count: descending - CountByCategoryLarge: - title: "[[this.short-name]] per [[GenericCategoryLarge]]" + title: "Per [[GenericCategoryLarge]]" dimensions: GenericCategoryLarge metrics: Count visualization: table diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index d618f74e4cbac..64fd4dd82d5b7 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -56,6 +56,9 @@ field/map->FieldInstance (classify/run-classifiers {}))))) +(def ^:private ^{:arglists '([root])} source-name + (comp (some-fn :display_name :name) :source)) + (def ^:private op->name {:sum (tru "sum") :avg (tru "average") @@ -81,7 +84,7 @@ filters/field-reference->id (->field root) :display_name) - (-> root :source :name)))) + (source-name root)))) (defn- join-enumeration [[x & xs]] @@ -127,7 +130,7 @@ (let [table (-> segment :table_id Table)] {:entity segment :full-name (tru "{0} in {1} segment" (:display_name table) (:name segment)) - :short-name (tru "these {0}" (:display_name table)) + :short-name (:display_name table) :source table :database (:db_id table) :query-filter (-> segment :definition :filter) @@ -188,26 +191,30 @@ (defmethod ->root (type Card) [card] - {:entity card - :source (source card) - :database (:database_id card) - :query-filter (qp.util/get-in-normalized card [:dataset_query :query :filter]) - :full-name (tru "\"{0}\" question" (:name card)) - :url (format "%squestion/%s" public-endpoint (u/get-id card)) - :rules-prefix [(if (table-like? card) - "table" - "question")]}) + (let [source (source card)] + {:entity card + :source source + :database (:database_id card) + :query-filter (qp.util/get-in-normalized card [:dataset_query :query :filter]) + :full-name (tru "\"{0}\" question" (:name card)) + :short-name (source-name {:source source}) + :url (format "%squestion/%s" public-endpoint (u/get-id card)) + :rules-prefix [(if (table-like? card) + "table" + "question")]})) (defmethod ->root (type Query) [query] - (let [source (source query)] + (let [source (source query)] {:entity query :source source :database (:database-id query) + :query-filter (qp.util/get-in-normalized query [:dataset_query :query :filter]) :full-name (cond (native-query? query) (tru "Native query") (table-like? query) (-> source ->root :full-name) :else (question-description {:source source} query)) + :short-name (source-name {:source source}) :url (format "%sadhoc/%s" public-endpoint (encode-base64-json query)) :rules-prefix [(if (table-like? query) "table" @@ -687,7 +694,7 @@ (-> rule (select-keys [:title :description :transient_title :groups]) (instantiate-metadata context {}) - (assoc :refinements (:cell-query root))))) + (assoc :refinements (filters/inject-refinement (:query-filter root) (:cell-query root)))))) (s/defn ^:private apply-rule [root, rule :- rules/Rule] @@ -701,9 +708,9 @@ (-> rule :cards nil?)) [(assoc dashboard :filters filters - :cards cards - :context context) - rule]))) + :cards cards) + rule + context]))) (def ^:private ^:const ^Long max-related 6) (def ^:private ^:const ^Long max-cards 15) @@ -739,16 +746,15 @@ [root, rule :- (s/maybe rules/Rule)] (->> (rules/get-rules (concat (:rules-prefix root) [(:rule rule)])) (keep (fn [indepth] - (when-let [[dashboard _] (apply-rule root indepth)] + (when-let [[dashboard _ _] (apply-rule root indepth)] {:title ((some-fn :short-title :title) dashboard) :description (:description dashboard) :url (format "%s/rule/%s/%s" (:url root) (:rule rule) (:rule indepth))}))) (take max-related))) (defn- drilldown-fields - [dashboard] - (->> dashboard - :context + [context] + (->> context :dimensions vals (mapcat :matches) @@ -756,127 +762,43 @@ (map ->related-entity))) (s/defn ^:private related - [dashboard, rule :- (s/maybe rules/Rule)] - (let [root (-> dashboard :context :root) + [context, rule :- (s/maybe rules/Rule)] + (let [root (:root context) indepth (indepth root rule)] (if (not-empty indepth) {:indepth indepth :related (related-entities (- max-related (count indepth)) root)} - (let [drilldown-fields (drilldown-fields dashboard) + (let [drilldown-fields (drilldown-fields context) n-related-entities (max (math/floor (* (/ 2 3) max-related)) (- max-related (count drilldown-fields)))] {:related (related-entities n-related-entities root) :drilldown-fields (take (- max-related n-related-entities) drilldown-fields)})))) -(def ^:private date-formatter (t.format/formatter "MMMM d, YYYY")) -(def ^:private datetime-formatter (t.format/formatter "EEEE, MMMM d, YYYY h:mm a")) - -(defn- humanize-datetime - [dt] - (t.format/unparse (if (str/index-of dt "T") - datetime-formatter - date-formatter) - (t.format/parse dt))) - -(defn- field-reference->field - [fieldset field-reference] - (cond-> (-> field-reference - filters/collect-field-references - first - filters/field-reference->id - fieldset) - (-> field-reference first qp.util/normalize-token (= :datetime-field)) - (assoc :unit (-> field-reference last qp.util/normalize-token)))) - -(defmulti - ^{:private true - :arglists '([fieldset [op & args]])} - humanize-filter-value (fn [_ [op & args]] - (qp.util/normalize-token op))) - -(def ^:private unit-name (comp {:minute-of-hour "minute of hour" - :hour-of-day "hour of day" - :day-of-week "day of week" - :day-of-month "day of month" - :week-of-year "week of year" - :month-of-year "month of year" - :quarter-of-year "quarter of year"} - qp.util/normalize-token)) - -(defn- field-name - ([fieldset field-reference] - (->> field-reference (field-reference->field fieldset) field-name)) - ([{:keys [display_name unit] :as field}] - (cond->> display_name - (and (filters/periodic-datetime? field) unit) (format "%s of %s" (unit-name unit))))) - -(defmethod humanize-filter-value := - [fieldset [_ field-reference value]] - (let [field (field-reference->field fieldset field-reference) - field-name (field-name field)] - (cond - (#{:type/State :type/Country} (:special_type field)) - (tru "in {0}" value) - - (filters/datetime? field) - (tru "where {0} is on {1}" field-name (humanize-datetime value)) - - :else - (tru "where {0} is {1}" field-name value)))) - -(defmethod humanize-filter-value :between - [fieldset [_ field-reference min-value max-value]] - (tru "where {0} is between {1} and {2}" (field-name fieldset field-reference) min-value max-value)) - -(defmethod humanize-filter-value :inside - [fieldset [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]] - (tru "where {0} is between {1} and {2}; and {3} is between {4} and {5}" - (field-name fieldset lon-reference) lon-min lon-max - (field-name fieldset lat-reference) lat-min lat-max)) - -(defn- cell-title - [context cell-query] - (let [source-name (-> context :root :source ((some-fn :display_name :name))) - fieldset (->> context - :tables - (mapcat :fields) - (map (fn [field] - [((some-fn :id :name) field) field])) - (into {}))] - (str/join " " [source-name (humanize-filter-value fieldset cell-query)]))) - (defn- automagic-dashboard "Create dashboards for table `root` using the best matching heuristics." - [{:keys [rule show rules-prefix query-filter cell-query full-name] :as root}] - (if-let [[dashboard rule] (if rule - (apply-rule root (rules/get-rule rule)) - (->> root - (matching-rules (rules/get-rules rules-prefix)) - (keep (partial apply-rule root)) - ;; `matching-rules` returns an `ArraySeq` (via `sort-by`) so - ;; `first` realises one element at a time (no chunking). - first))] + [{:keys [rule show rules-prefix full-name] :as root}] + (if-let [[dashboard rule context] (if rule + (apply-rule root (rules/get-rule rule)) + (->> root + (matching-rules (rules/get-rules rules-prefix)) + (keep (partial apply-rule root)) + ;; `matching-rules` returns an `ArraySeq` (via `sort-by`) + ;; so `first` realises one element at a time + ;; (no chunking). + first))] (do (log/infof (trs "Applying heuristic %s to %s.") (:rule rule) full-name) (log/infof (trs "Dimensions bindings:\n%s") - (->> dashboard - :context + (->> context :dimensions (m/map-vals #(update % :matches (partial map :name))) u/pprint-to-str)) (log/infof (trs "Using definitions:\nMetrics:\n%s\nFilters:\n%s") - (-> dashboard :context :metrics u/pprint-to-str) - (-> dashboard :context :filters u/pprint-to-str)) - (-> (cond-> dashboard - cell-query - (assoc :transient_title nil - :title (tru "A closer look at {0}" (cell-title (:context dashboard) - cell-query))) - - query-filter - (assoc :title (tru "A closer look at {0}" (:full-name root)))) + (-> context :metrics u/pprint-to-str) + (-> context :filters u/pprint-to-str)) + (-> dashboard (populate/create-dashboard (or show max-cards)) - (assoc :related (related dashboard rule) + (assoc :related (related context rule) :more (when (and (-> dashboard :cards count (> max-cards)) (not= show :all)) (format "%s#show=all" (:url root)))))) @@ -934,50 +856,125 @@ (concat (collect-metrics root question) (collect-breakout-fields root question)))) +(def ^:private date-formatter (t.format/formatter "MMMM d, YYYY")) +(def ^:private datetime-formatter (t.format/formatter "EEEE, MMMM d, YYYY h:mm a")) + +(defn- humanize-datetime + [dt] + (t.format/unparse (if (str/index-of dt "T") + datetime-formatter + date-formatter) + (t.format/parse dt))) + +(defn- field-reference->field + [root field-reference] + (cond-> (->> field-reference + filters/collect-field-references + first + filters/field-reference->id + (->field root)) + (-> field-reference first qp.util/normalize-token (= :datetime-field)) + (assoc :unit (-> field-reference last qp.util/normalize-token)))) + +(defmulti + ^{:private true + :arglists '([fieldset [op & args]])} + humanize-filter-value (fn [_ [op & args]] + (qp.util/normalize-token op))) + +(def ^:private unit-name (comp {:minute-of-hour "minute of hour" + :hour-of-day "hour of day" + :day-of-week "day of week" + :day-of-month "day of month" + :week-of-year "week of year" + :month-of-year "month of year" + :quarter-of-year "quarter of year"} + qp.util/normalize-token)) + +(defn- field-name + ([root field-reference] + (->> field-reference (field-reference->field root) field-name)) + ([{:keys [display_name unit] :as field}] + (cond->> display_name + (and (filters/periodic-datetime? field) unit) (format "%s of %s" (unit-name unit))))) + +(defmethod humanize-filter-value := + [root [_ field-reference value]] + (let [field (field-reference->field root field-reference) + field-name (field-name field)] + (cond + (#{:type/State :type/Country} (:special_type field)) + (tru "in {0}" value) + + (filters/datetime? field) + (tru "where {0} is on {1}" field-name (humanize-datetime value)) + + :else + (tru "where {0} is {1}" field-name value)))) + +(defmethod humanize-filter-value :between + [root [_ field-reference min-value max-value]] + (tru "where {0} is between {1} and {2}" (field-name root field-reference) min-value max-value)) + +(defmethod humanize-filter-value :inside + [root [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]] + (tru "where {0} is between {1} and {2}; and {3} is between {4} and {5}" + (field-name root lon-reference) lon-min lon-max + (field-name root lat-reference) lat-min lat-max)) + +(defn- cell-title + [root cell-query] + (str/join " " [(->> (qp.util/get-in-normalized (-> root :entity) [:dataset_query :query :aggregation]) + (map (partial metric->description root)) + join-enumeration) + (humanize-filter-value root cell-query)])) + (defmethod automagic-analysis (type Card) [card {:keys [cell-query] :as opts}] - (let [root (->root card)] - (if (or (table-like? card) - cell-query) + (let [root (->root card) + cell-url (format "%squestion/%s/cell/%s" public-endpoint + (u/get-id card) + (encode-base64-json cell-query))] + (if (table-like? card) (automagic-dashboard (merge (cond-> root - cell-query (merge {:url (format "%squestion/%s/cell/%s" public-endpoint - (u/get-id card) - (encode-base64-json cell-query)) + cell-query (merge {:url cell-url :entity (:source root) - :short-name (->> root - :source - ((some-fn :display_name :name)) - (tru "these {0}")) :rules-prefix ["table"]})) opts)) (let [opts (assoc opts :show :all)] - (->> (decompose-question root card opts) - (apply populate/merge-dashboards (automagic-dashboard root)) - (merge {:related (related {:context {:root {:entity card}}} nil)})))))) + (cond-> (apply populate/merge-dashboards + (automagic-dashboard (merge (cond-> root + cell-query (assoc :url cell-url)) + opts)) + (decompose-question root card opts)) + cell-query (merge (let [title (tru "A closer look at {0}" (cell-title root cell-query))] + {:transient_name title + :name title}))))))) (defmethod automagic-analysis (type Query) [query {:keys [cell-query] :as opts}] - (let [root (->root query)] - (if (or (table-like? query) - (:cell-query opts)) + (let [root (->root query) + cell-url (format "%sadhoc/%s/cell/%s" public-endpoint + (encode-base64-json (:dataset_query query)) + (encode-base64-json cell-query))] + (if (table-like? query) (automagic-dashboard (merge (cond-> root - cell-query (merge {:url (format "%sadhoc/%s/cell/%s" public-endpoint - (encode-base64-json (:dataset_query query)) - (encode-base64-json cell-query)) + cell-query (merge {:url cell-url :entity (:source root) :query-filter (qp.util/get-in-normalized query [:dataset_query :query :filter]) - :short-name (->> root - :source - ((some-fn :display_name :name)) - (tru "these {0}")) :rules-prefix ["table"]})) opts)) (let [opts (assoc opts :show :all)] - (->> (decompose-question root query opts) - (apply populate/merge-dashboards (automagic-dashboard root)) - (merge {:related (related {:context {:root {:entity query}}} nil)})))))) + (cond-> (apply populate/merge-dashboards + (automagic-dashboard (merge (cond-> root + cell-query (assoc :url cell-url)) + opts)) + (decompose-question root query opts)) + cell-query (merge (let [title (tru "A closer look at {0}" (cell-title root cell-query))] + {:transient_name title + :name title}))))))) (defmethod automagic-analysis (type Field) [field opts] @@ -990,8 +987,7 @@ :from [Field] :where [:in :table_id (map u/get-id tables)] :group-by [:table_id]}) - (into {} (map (fn [{:keys [count table_id]}] - [table_id count])))) + (into {} (map (juxt :table_id count)))) list-like? (->> (when-let [candidates (->> field-count (filter (comp (partial >= 2) val)) (map key) From 36fa7eb4b9c07aac20b4ef12874b82a93adf5d84 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Tue, 26 Jun 2018 18:45:43 +0200 Subject: [PATCH 014/116] downsize titles when merging dashboards --- .../automagic_dashboards/populate.clj | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/metabase/automagic_dashboards/populate.clj b/src/metabase/automagic_dashboards/populate.clj index bf671adab3cc2..dadc286cd7b53 100644 --- a/src/metabase/automagic_dashboards/populate.clj +++ b/src/metabase/automagic_dashboards/populate.clj @@ -8,6 +8,7 @@ [card :as card] [field :refer [Field]]] [metabase.query-processor.util :as qp.util] + [metabase.util :as u] [puppetlabs.i18n.core :as i18n :refer [trs]] [toucan.db :as db])) @@ -265,6 +266,16 @@ (cond-> dashboard (not-empty filters) (filters/add-filters filters max-filters))))) +(defn- downsize-titles + [markdown] + (->> markdown + str/split-lines + (map (fn [line] + (if (str/starts-with? line "#") + (str "#" line) + line))) + str/join)) + (defn merge-dashboards "Merge dashboards `ds` into dashboard `d`." [d & ds] @@ -288,7 +299,14 @@ (map #(+ (:row %) (:sizeY %))) (apply max -1) ; -1 so it neturalizes +1 for spacing if ; the target dashboard is empty. - inc)] + inc) + cards (->> dashboard + :ordered_cards + (map #(-> % + (update :row + offset group-heading-height) + (u/update-in-when [:visualization_settings :text] + downsize-titles) + (dissoc :parameter_mappings))))] (-> target (add-text-card {:width grid-width :height group-heading-height @@ -296,12 +314,7 @@ :visualization-settings {:dashcard.background false :text.align_vertical :bottom}} [offset 0]) - (update :ordered_cards concat - (->> dashboard - :ordered_cards - (map #(-> % - (update :row + offset group-heading-height) - (dissoc :parameter_mappings)))))))) + (update :ordered_cards concat cards)))) d ds) (not-empty filter-targets) (filters/add-filters filter-targets max-filters)))) From 0712bfc79deb05344408db3daac84b8b8e4b1a47 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Tue, 26 Jun 2018 19:00:55 +0200 Subject: [PATCH 015/116] Fix enumeration of field names in titles --- src/metabase/automagic_dashboards/core.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 64fd4dd82d5b7..f84410819091c 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -87,10 +87,10 @@ (source-name root)))) (defn- join-enumeration - [[x & xs]] - (if xs + [xs] + (if (next xs) (tru "{0} and {1}" (str/join ", " (butlast xs)) (last xs)) - x)) + (first xs))) (defn- question-description [root question] From c717e8dd7e5e22848c8b705791cccb56ff943810 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Tue, 26 Jun 2018 19:19:36 +0200 Subject: [PATCH 016/116] Correctly handle cell xrays of questions with multiple breakout dims --- src/metabase/automagic_dashboards/core.clj | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index f84410819091c..9cec02b16eda7 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -902,32 +902,32 @@ [root [_ field-reference value]] (let [field (field-reference->field root field-reference) field-name (field-name field)] - (cond - (#{:type/State :type/Country} (:special_type field)) - (tru "in {0}" value) - - (filters/datetime? field) - (tru "where {0} is on {1}" field-name (humanize-datetime value)) - - :else - (tru "where {0} is {1}" field-name value)))) + (if (filters/datetime? field) + (tru "{0} is on {1}" field-name (humanize-datetime value)) + (tru "{0} is {1}" field-name value)))) (defmethod humanize-filter-value :between [root [_ field-reference min-value max-value]] - (tru "where {0} is between {1} and {2}" (field-name root field-reference) min-value max-value)) + (tru "{0} is between {1} and {2}" (field-name root field-reference) min-value max-value)) (defmethod humanize-filter-value :inside [root [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]] - (tru "where {0} is between {1} and {2}; and {3} is between {4} and {5}" + (tru "{0} is between {1} and {2}; and {3} is between {4} and {5}" (field-name root lon-reference) lon-min lon-max (field-name root lat-reference) lat-min lat-max)) +(defmethod humanize-filter-value :and + [root [_ & clauses]] + (->> clauses + (map (partial humanize-filter-value root)) + join-enumeration)) + (defn- cell-title [root cell-query] (str/join " " [(->> (qp.util/get-in-normalized (-> root :entity) [:dataset_query :query :aggregation]) (map (partial metric->description root)) join-enumeration) - (humanize-filter-value root cell-query)])) + (tru "where {0}" (humanize-filter-value root cell-query))])) (defmethod automagic-analysis (type Card) [card {:keys [cell-query] :as opts}] From 853a9ed61bb8be479a29e06940d15be0eabd58ae Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Wed, 27 Jun 2018 19:20:15 +0200 Subject: [PATCH 017/116] Fix regression in candidates --- src/metabase/automagic_dashboards/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 9cec02b16eda7..3c2793f222258 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -987,7 +987,7 @@ :from [Field] :where [:in :table_id (map u/get-id tables)] :group-by [:table_id]}) - (into {} (map (juxt :table_id count)))) + (into {} (map (juxt :table_id :count)))) list-like? (->> (when-let [candidates (->> field-count (filter (comp (partial >= 2) val)) (map key) From 5e15d80f2879f830f7b0aaea9a992d499b9475db Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 28 Jun 2018 10:10:44 +0200 Subject: [PATCH 018/116] Fix broken url generation --- src/metabase/automagic_dashboards/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 3c2793f222258..34607383dbaea 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -215,7 +215,7 @@ (table-like? query) (-> source ->root :full-name) :else (question-description {:source source} query)) :short-name (source-name {:source source}) - :url (format "%sadhoc/%s" public-endpoint (encode-base64-json query)) + :url (format "%sadhoc/%s" public-endpoint (encode-base64-json (:dataset_query query))) :rules-prefix [(if (table-like? query) "table" "question")]})) From 5b5ec2912ab8310e7ca0aae6e584b5f106bf3e6d Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 28 Jun 2018 10:53:56 +0200 Subject: [PATCH 019/116] Test for more working correctly --- src/metabase/automagic_dashboards/core.clj | 8 +-- .../automagic_dashboards/core_test.clj | 56 ++++++++++--------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 34607383dbaea..2a3332399601c 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -786,7 +786,7 @@ ;; so `first` realises one element at a time ;; (no chunking). first))] - (do + (let [show (or show max-cards)] (log/infof (trs "Applying heuristic %s to %s.") (:rule rule) full-name) (log/infof (trs "Dimensions bindings:\n%s") (->> context @@ -797,10 +797,10 @@ (-> context :metrics u/pprint-to-str) (-> context :filters u/pprint-to-str)) (-> dashboard - (populate/create-dashboard (or show max-cards)) + (populate/create-dashboard show) (assoc :related (related context rule) - :more (when (and (-> dashboard :cards count (> max-cards)) - (not= show :all)) + :more (when (and (not= show :all) + (-> dashboard :cards count (> show))) (format "%s#show=all" (:url root)))))) (throw (ex-info (trs "Can''t create dashboard for {0}" full-name) {:root root diff --git a/test/metabase/automagic_dashboards/core_test.clj b/test/metabase/automagic_dashboards/core_test.clj index 88fa19a03c700..d1fff9f4932e5 100644 --- a/test/metabase/automagic_dashboards/core_test.clj +++ b/test/metabase/automagic_dashboards/core_test.clj @@ -82,7 +82,7 @@ (tree-seq (some-fn sequential? map?) identity) (keep (fn [form] (when (map? form) - (:url form)))))) + ((some-fn :url :more) form)))))) (defn- valid-urls? [dashboard] @@ -103,12 +103,21 @@ (assert (every? valid-card? (keep :card (:ordered_cards dashboard)))) true) +(defn- test-automagic-analysis + ([entity] (test-automagic-analysis entity nil)) + ([entity cell-query] + ;; We want to both generate as many cards as we can to catch all aberrations, but also make sure + ;; that size limiting works. + (and (valid-dashboard? (automagic-analysis entity {:cell-query cell-query :show :all})) + (let [dashboard (automagic-analysis entity {:cell-query cell-query :show 1})] + (assert (->> dashboard :ordered_cards (keep :card) count (>= 1))) + (valid-dashboard? dashboard))))) + (expect (with-rasta (with-dashboard-cleanup (->> (db/select Table :db_id (data/id)) - (keep #(automagic-analysis % {})) - (every? valid-dashboard?))))) + (every? test-automagic-analysis))))) (expect (with-rasta @@ -116,15 +125,14 @@ (->> (db/select Field :table_id [:in (db/select-field :id Table :db_id (data/id))] :visibility_type "normal") - (keep #(automagic-analysis % {})) - (every? valid-dashboard?))))) + (every? test-automagic-analysis))))) (expect (tt/with-temp* [Metric [{metric-id :id} {:table_id (data/id :venues) :definition {:query {:aggregation ["count"]}}}]] (with-rasta (with-dashboard-cleanup - (->> (Metric) (keep #(automagic-analysis % {})) (every? valid-dashboard?)))))) + (->> (Metric) (every? test-automagic-analysis)))))) (expect (tt/with-temp* [Card [{card-id :id} {:table_id (data/id :venues) @@ -134,7 +142,7 @@ :database (data/id)}}]] (with-rasta (with-dashboard-cleanup - (-> card-id Card (automagic-analysis {}) valid-dashboard?))))) + (-> card-id Card test-automagic-analysis))))) (expect (tt/with-temp* [Card [{card-id :id} {:table_id (data/id :venues) @@ -145,7 +153,7 @@ :database (data/id)}}]] (with-rasta (with-dashboard-cleanup - (-> card-id Card (automagic-analysis {}) valid-dashboard?))))) + (-> card-id Card test-automagic-analysis))))) (expect (tt/with-temp* [Card [{card-id :id} {:table_id nil @@ -154,7 +162,7 @@ :database (data/id)}}]] (with-rasta (with-dashboard-cleanup - (-> card-id Card (automagic-analysis {}) valid-dashboard?))))) + (-> card-id Card test-automagic-analysis))))) (expect (tt/with-temp* [Card [{source-id :id} {:table_id (data/id :venues) @@ -168,7 +176,7 @@ :database -1337}}]] (with-rasta (with-dashboard-cleanup - (-> card-id Card (automagic-analysis {}) valid-dashboard?))))) + (-> card-id Card test-automagic-analysis))))) (expect (tt/with-temp* [Card [{source-id :id} {:table_id nil @@ -182,7 +190,7 @@ :database -1337}}]] (with-rasta (with-dashboard-cleanup - (-> card-id Card (automagic-analysis {}) valid-dashboard?))))) + (-> card-id Card test-automagic-analysis))))) (expect (tt/with-temp* [Card [{card-id :id} {:table_id nil @@ -191,7 +199,7 @@ :database (data/id)}}]] (with-rasta (with-dashboard-cleanup - (-> card-id Card (automagic-analysis {}) valid-dashboard?))))) + (-> card-id Card test-automagic-analysis))))) (expect (tt/with-temp* [Card [{card-id :id} {:table_id (data/id :venues) @@ -201,10 +209,9 @@ :database (data/id)}}]] (with-rasta (with-dashboard-cleanup - (-> card-id - Card - (automagic-analysis {:cell-query [:= [:field-id (data/id :venues :category_id)] 2]}) - valid-dashboard?))))) + (->> card-id + Card + (test-automagic-analysis [:= [:field-id (data/id :venues :category_id)] 2])))))) (expect @@ -215,10 +222,9 @@ :database (data/id)}}]] (with-rasta (with-dashboard-cleanup - (-> card-id - Card - (automagic-analysis {:cell-query [:= [:field-id (data/id :venues :category_id)] 2]}) - valid-dashboard?))))) + (->> card-id + Card + (test-automagic-analysis [:= [:field-id (data/id :venues :category_id)] 2])))))) (expect @@ -228,7 +234,7 @@ :source_table (data/id :venues)} :type :query :database (data/id)})] - (-> q (automagic-analysis {}) valid-dashboard?))))) + (test-automagic-analysis q))))) (expect (with-rasta @@ -238,7 +244,7 @@ :source_table (data/id :venues)} :type :query :database (data/id)})] - (-> q (automagic-analysis {}) valid-dashboard?))))) + (test-automagic-analysis q))))) (expect (with-rasta @@ -248,7 +254,7 @@ :source_table (data/id :checkins)} :type :query :database (data/id)})] - (-> q (automagic-analysis {}) valid-dashboard?))))) + (test-automagic-analysis q))))) (expect (with-rasta @@ -257,9 +263,7 @@ :source_table (data/id :venues)} :type :query :database (data/id)})] - (-> q - (automagic-analysis {:cell-query [:= [:field-id (data/id :venues :category_id)] 2]}) - valid-dashboard?))))) + (test-automagic-analysis q [:= [:field-id (data/id :venues :category_id)] 2]))))) ;;; ------------------- /candidates ------------------- From ce92d45e78952e613d4a26d901ccb9a673e7f5c3 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 28 Jun 2018 12:40:08 +0200 Subject: [PATCH 020/116] Don't show cards and filters where breakdown dim is a cell dim --- src/metabase/automagic_dashboards/core.clj | 50 ++++++++++++++------- src/metabase/automagic_dashboards/rules.clj | 4 +- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 2a3332399601c..887d0fdd3852b 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -531,6 +531,19 @@ (not (and (isa? base_type :type/Number) (= engine :druid)))) +(defn- singular-cell-dimensions + [root] + (letfn [(collect-dimensions [[op & args]] + (case (qp.util/normalize-token op) + :and (mapcat collect-dimensions args) + := (filters/collect-field-references args) + nil))] + (->> root + :cell-query + collect-dimensions + (map filters/field-reference->id) + set))) + (defn- card-candidates "Generate all potential cards given a card definition and bindings for dimensions, metrics, and filters." @@ -549,25 +562,27 @@ rules/max-score) (/ score rules/max-score))) dimensions (map (comp (partial into [:dimension]) first) dimensions) - used-dimensions (rules/collect-dimensions [dimensions metrics filters query])] + used-dimensions (rules/collect-dimensions [dimensions metrics filters query]) + cell-dimension? (->> context :root singular-cell-dimensions)] (->> used-dimensions (map (some-fn #(get-in (:dimensions context) [% :matches]) (comp #(filter-tables % (:tables context)) rules/->entity))) (apply combo/cartesian-product) - (filter (fn [instantiations] + (map (partial zipmap used-dimensions)) + (filter (fn [bindings] (->> dimensions - (map (comp (zipmap used-dimensions instantiations) second)) - (every? valid-breakout-dimension?)))) - (map (fn [instantiations] - (let [bindings (zipmap used-dimensions instantiations) - query (if query - (build-query context bindings query) - (build-query context bindings - filters - metrics - dimensions - limit - order_by))] + (map (comp bindings second)) + (every? (every-pred valid-breakout-dimension? + (complement (comp cell-dimension? id-or-name))))))) + (map (fn [bindings] + (let [query (if query + (build-query context bindings query) + (build-query context bindings + filters + metrics + dimensions + limit + order_by))] (-> card (instantiate-metadata context (->> metrics (map :name) @@ -702,13 +717,14 @@ dashboard (make-dashboard root rule context) filters (->> rule :dashboard_filters - (mapcat (comp :matches (:dimensions context)))) + (mapcat (comp :matches (:dimensions context))) + (remove (comp (singular-cell-dimensions root) id-or-name))) cards (make-cards context rule)] (when (or (not-empty cards) (-> rule :cards nil?)) [(assoc dashboard - :filters filters - :cards cards) + :filters filters + :cards cards) rule context]))) diff --git a/src/metabase/automagic_dashboards/rules.clj b/src/metabase/automagic_dashboards/rules.clj index 50bad7029869e..6e6842033f60c 100644 --- a/src/metabase/automagic_dashboards/rules.clj +++ b/src/metabase/automagic_dashboards/rules.clj @@ -4,6 +4,7 @@ [clojure.string :as str] [clojure.tools.logging :as log] [metabase.automagic-dashboards.populate :as populate] + [metabase.query-processor.util :as qp.util] [metabase.util :as u] [metabase.util.schema :as su] [puppetlabs.i18n.core :as i18n :refer [trs]] @@ -119,8 +120,7 @@ (mapcat (comp k val first) cards)) (def ^:private DimensionForm - [(s/one (s/constrained (s/cond-pre s/Str s/Keyword) - (comp #{"dimension"} str/lower-case name)) + [(s/one (s/constrained (s/cond-pre s/Str s/Keyword) (comp #{:dimension} qp.util/normalize-token)) "dimension") (s/one s/Str "identifier") su/Map]) From 9e9744a96009c9d91da52acdfea549c70f7dea39 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 28 Jun 2018 19:05:08 +0200 Subject: [PATCH 021/116] don't explode if cell-query is nil --- src/metabase/automagic_dashboards/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 887d0fdd3852b..5388ed90bbb46 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -534,7 +534,7 @@ (defn- singular-cell-dimensions [root] (letfn [(collect-dimensions [[op & args]] - (case (qp.util/normalize-token op) + (case (some-> op qp.util/normalize-token) :and (mapcat collect-dimensions args) := (filters/collect-field-references args) nil))] From c6543c167098bf3db1687cc7198ffc6bee892270 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 28 Jun 2018 20:07:08 +0200 Subject: [PATCH 022/116] Correct title for questions with no breakout dimension --- src/metabase/automagic_dashboards/core.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 5388ed90bbb46..d13a61218e489 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -103,7 +103,9 @@ (partial ->field root) filters/field-reference->id)) join-enumeration)] - (tru "{0} by {1}" aggregations dimensions))) + (if dimensions + (tru "{0} by {1}" aggregations dimensions) + aggregations))) (def ^:private ^{:arglists '([x])} encode-base64-json (comp codec/base64-encode codecs/str->bytes json/encode)) From e7e189ba212a0eb8ad462f76ae44fbb9dd139d0e Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 28 Jun 2018 20:26:19 +0200 Subject: [PATCH 023/116] tests: set up collection for cards so that permissions work --- .../automagic_dashboards/core_test.clj | 60 ++++++++++++++----- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/test/metabase/automagic_dashboards/core_test.clj b/test/metabase/automagic_dashboards/core_test.clj index d1fff9f4932e5..42534e2572ebd 100644 --- a/test/metabase/automagic_dashboards/core_test.clj +++ b/test/metabase/automagic_dashboards/core_test.clj @@ -6,9 +6,12 @@ [rules :as rules]] [metabase.models [card :refer [Card]] + [collection :refer [Collection]] [database :refer [Database]] [field :as field :refer [Field]] [metric :refer [Metric]] + [permissions :as perms] + [permissions-group :as perms-group] [query :as query] [table :refer [Table] :as table] [user :as user]] @@ -89,8 +92,8 @@ (->> dashboard collect-urls (every? (fn [url] - ((test-users/user->client :rasta) :get 200 (format "automagic-dashboards/%s" - (subs url 16))))))) + ((test-users/user->client :rasta) :get 200 + (format "automagic-dashboards/%s" (subs url 16))))))) (def ^:private valid-card? (comp qp/expand :dataset_query)) @@ -135,17 +138,22 @@ (->> (Metric) (every? test-automagic-analysis)))))) (expect - (tt/with-temp* [Card [{card-id :id} {:table_id (data/id :venues) + (tt/with-temp* [Collection [{collection-id :id}] + Card [{card-id :id} {:table_id (data/id :venues) + :collection_id collection-id :dataset_query {:query {:filter [:> [:field-id (data/id :venues :price)] 10] :source_table (data/id :venues)} :type :query :database (data/id)}}]] (with-rasta (with-dashboard-cleanup + (perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) (-> card-id Card test-automagic-analysis))))) (expect - (tt/with-temp* [Card [{card-id :id} {:table_id (data/id :venues) + (tt/with-temp* [Collection [{collection-id :id}] + Card [{card-id :id} {:table_id (data/id :venues) + :collection_id collection-id :dataset_query {:query {:aggregation [[:count]] :breakout [[:field-id (data/id :venues :category_id)]] :source_table (data/id :venues)} @@ -156,75 +164,95 @@ (-> card-id Card test-automagic-analysis))))) (expect - (tt/with-temp* [Card [{card-id :id} {:table_id nil + (tt/with-temp* [Collection [{collection-id :id}] + Card [{card-id :id} {:table_id nil + :collection_id collection-id :dataset_query {:native {:query "select * from users"} :type :native :database (data/id)}}]] (with-rasta (with-dashboard-cleanup + (perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) (-> card-id Card test-automagic-analysis))))) (expect - (tt/with-temp* [Card [{source-id :id} {:table_id (data/id :venues) + (tt/with-temp* [Collection [{collection-id :id}] + Card [{source-id :id} {:table_id (data/id :venues) + :collection_id collection-id :dataset_query {:query {:source_table (data/id :venues)} :type :query :database (data/id)}}] Card [{card-id :id} {:table_id (data/id :venues) + :collection_id collection-id :dataset_query {:query {:filter [:> [:field-id (data/id :venues :price)] 10] :source_table (str "card__" source-id)} :type :query :database -1337}}]] (with-rasta (with-dashboard-cleanup + (perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) (-> card-id Card test-automagic-analysis))))) (expect - (tt/with-temp* [Card [{source-id :id} {:table_id nil + (tt/with-temp* [Collection [{collection-id :id}] + Card [{source-id :id} {:table_id nil + :collection_id collection-id :dataset_query {:native {:query "select * from users"} :type :native :database (data/id)}}] Card [{card-id :id} {:table_id (data/id :venues) + :collection_id collection-id :dataset_query {:query {:filter [:> [:field-id (data/id :venues :price)] 10] :source_table (str "card__" source-id)} :type :query :database -1337}}]] (with-rasta (with-dashboard-cleanup + (perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) (-> card-id Card test-automagic-analysis))))) (expect - (tt/with-temp* [Card [{card-id :id} {:table_id nil + (tt/with-temp* [Collection [{collection-id :id}] + Card [{card-id :id} {:table_id nil + :collection_id collection-id :dataset_query {:native {:query "select * from users"} :type :native :database (data/id)}}]] (with-rasta (with-dashboard-cleanup + (perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) (-> card-id Card test-automagic-analysis))))) (expect - (tt/with-temp* [Card [{card-id :id} {:table_id (data/id :venues) + (tt/with-temp* [Collection [{collection-id :id}] + Card [{card-id :id} {:table_id (data/id :venues) + :collection_id collection-id :dataset_query {:query {:filter [:> [:field-id (data/id :venues :price)] 10] :source_table (data/id :venues)} :type :query :database (data/id)}}]] (with-rasta (with-dashboard-cleanup - (->> card-id - Card - (test-automagic-analysis [:= [:field-id (data/id :venues :category_id)] 2])))))) + (perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) + (-> card-id + Card + (test-automagic-analysis [:= [:field-id (data/id :venues :category_id)] 2])))))) (expect - (tt/with-temp* [Card [{card-id :id} {:table_id (data/id :venues) + (tt/with-temp* [Collection [{collection-id :id}] + Card [{card-id :id} {:table_id (data/id :venues) + :collection_id collection-id :dataset_query {:query {:filter [:> [:field-id (data/id :venues :price)] 10] :source_table (data/id :venues)} :type :query :database (data/id)}}]] (with-rasta (with-dashboard-cleanup - (->> card-id - Card - (test-automagic-analysis [:= [:field-id (data/id :venues :category_id)] 2])))))) + (perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) + (-> card-id + Card + (test-automagic-analysis [:= [:field-id (data/id :venues :category_id)] 2])))))) (expect From 37f1c6e904df9f599f9217501ba11921de6ddda5 Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Thu, 28 Jun 2018 12:01:03 -0700 Subject: [PATCH 024/116] remove use of 'these' --- .../metric/GenericMetric.yaml | 6 +- .../question/GenericQuestion.yaml | 4 +- .../table/GenericTable.yaml | 64 +++++++++---------- src/metabase/automagic_dashboards/core.clj | 2 +- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/resources/automagic_dashboards/metric/GenericMetric.yaml b/resources/automagic_dashboards/metric/GenericMetric.yaml index 6af5a040a0d81..1968ab203fd16 100644 --- a/resources/automagic_dashboards/metric/GenericMetric.yaml +++ b/resources/automagic_dashboards/metric/GenericMetric.yaml @@ -1,5 +1,5 @@ title: A look at the [[this]] -transient_title: Here's a quick look at your [[this]] +transient_title: Here's a quick look at the [[this]] description: How it's distributed across time and other categories. applies_to: GenericTable metrics: @@ -39,9 +39,9 @@ groups: - Geographical: title: The [[this]] by location - Categories: - title: How the [[this]] is distributed across different categories + title: How this metric is distributed across different categories - Numbers: - title: How the [[this]] is distributed across different numbers + title: How this metric is distributed across different numbers - LargeCategories: title: The top and bottom for the [[this]] dashboard_filters: diff --git a/resources/automagic_dashboards/question/GenericQuestion.yaml b/resources/automagic_dashboards/question/GenericQuestion.yaml index a7d0cfaa713b0..ad7a7b8b067df 100644 --- a/resources/automagic_dashboards/question/GenericQuestion.yaml +++ b/resources/automagic_dashboards/question/GenericQuestion.yaml @@ -1,4 +1,4 @@ title: "A closer look at your [[this]]" transient_title: "Here's a closer look at your [[this]]" -description: Here is breakdown of metrics and dimensions used in [[this]]. -applies_to: GenericTable \ No newline at end of file +description: A closer look at the metrics and dimensions used in this saved question. +applies_to: GenericTable diff --git a/resources/automagic_dashboards/table/GenericTable.yaml b/resources/automagic_dashboards/table/GenericTable.yaml index 60f04bee42417..5688f39d7df89 100644 --- a/resources/automagic_dashboards/table/GenericTable.yaml +++ b/resources/automagic_dashboards/table/GenericTable.yaml @@ -112,7 +112,7 @@ dashboard_filters: cards: # Overview - Rowcount: - title: The number of [[this.short-name]] + title: Total [[this.short-name]] visualization: scalar metrics: Count score: 100 @@ -132,7 +132,7 @@ cards: group: Overview # General - NumberDistribution: - title: How these [[this.short-name]] are distributed across [[GenericNumber]] + title: How [[this.short-name]] are distributed across [[GenericNumber]] dimensions: - GenericNumber: aggregation: default @@ -141,7 +141,7 @@ cards: score: 90 group: General - CountByCategoryMedium: - title: "These [[this.short-name]] per [[GenericCategoryMedium]]" + title: "[[this.short-name]] per [[GenericCategoryMedium]]" dimensions: GenericCategoryMedium metrics: Count visualization: row @@ -151,7 +151,7 @@ cards: order_by: - Count: descending - CountByCategoryLarge: - title: "These [[this.short-name]] per [[GenericCategoryLarge]]" + title: "[[this.short-name]] per [[GenericCategoryLarge]]" dimensions: GenericCategoryLarge metrics: Count visualization: table @@ -162,7 +162,7 @@ cards: - Count: descending # Geographical - CountByCountry: - title: "These [[this.short-name]] per country" + title: "[[this.short-name]] per country" metrics: Count dimensions: Country score: 90 @@ -173,7 +173,7 @@ cards: group: Geographical height: 6 - CountByState: - title: "These [[this.short-name]] per state" + title: "[[this.short-name]] per state" metrics: Count dimensions: State score: 90 @@ -184,7 +184,7 @@ cards: group: Geographical height: 6 - CountByCoords: - title: "These [[this.short-name]] by coordinates" + title: "[[this.short-name]] by coordinates" metrics: Count dimensions: - Long @@ -195,14 +195,14 @@ cards: height: 6 # By Time - CountByJoinDate: - title: "These [[this.short-name]] that have joined over time" + title: "[[this.short-name]] that have joined over time" visualization: line dimensions: JoinTimestamp metrics: Count score: 90 group: ByTime - CountByJoinDate: - title: "These [[this.short-name]] that have joined over time" + title: "[[this.short-name]] that have joined over time" visualization: line dimensions: JoinDate metrics: Count @@ -223,21 +223,21 @@ cards: score: 90 group: ByTime - CountByTimestamp: - title: "These [[this.short-name]] by [[Timestamp]]" + title: "[[this.short-name]] by [[Timestamp]]" visualization: line dimensions: Timestamp metrics: Count score: 20 group: ByTime - CountByTimestamp: - title: "These [[this.short-name]] by [[Timestamp]]" + title: "[[this.short-name]] by [[Timestamp]]" visualization: line dimensions: Date metrics: Count score: 20 group: ByTime - NumberOverTime: - title: "These [[GenericNumber]] over time" + title: "[[GenericNumber]] over time" visualization: line dimensions: Timestamp metrics: @@ -371,7 +371,7 @@ cards: group: ByTime x_label: "[[Timestamp]]" - DayOfWeekCreateDate: - title: Weekdays when these [[this.short-name]] were added + title: Weekdays when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -381,7 +381,7 @@ cards: group: ByTime x_label: Created At by day of the week - DayOfWeekCreateDate: - title: Weekdays when these [[this.short-name]] were added + title: Weekdays when [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -391,7 +391,7 @@ cards: group: ByTime x_label: Created At by day of the week - HourOfDayCreateDate: - title: Hours when these [[this.short-name]] were added + title: Hours when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -401,7 +401,7 @@ cards: group: ByTime x_label: Created At by hour of the day - HourOfDayCreateDate: - title: Hours when these [[this.short-name]] were added + title: Hours when [[this.short-name]] were added visualization: bar dimensions: - CreateTime: @@ -411,7 +411,7 @@ cards: group: ByTime x_label: Created At by hour of the day - DayOfMonthCreateDate: - title: Days when these [[this.short-name]] were added + title: Days when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -421,7 +421,7 @@ cards: group: ByTime x_label: Created At by day of the month - DayOfMonthCreateDate: - title: Days when these [[this.short-name]] were added + title: Days when [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -431,7 +431,7 @@ cards: group: ByTime x_label: Created At by day of the month - MonthOfYearCreateDate: - title: Months when these [[this.short-name]] were added + title: Months when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -441,7 +441,7 @@ cards: group: ByTime x_label: Created At by month of the year - MonthOfYearCreateDate: - title: Months when these [[this.short-name]] were added + title: Months when [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -451,7 +451,7 @@ cards: group: ByTime x_label: Created At by month of the year - QuerterOfYearCreateDate: - title: Quarters when these [[this.short-name]] were added + title: Quarters when [[this.short-name]] were added visualization: bar dimensions: - CreateTimestamp: @@ -461,7 +461,7 @@ cards: group: ByTime x_label: Created At by quarter of the year - QuerterOfYearCreateDate: - title: Quarters when these [[this.short-name]] were added + title: Quarters when [[this.short-name]] were added visualization: bar dimensions: - CreateDate: @@ -471,7 +471,7 @@ cards: group: ByTime x_label: Created At by quarter of the year - DayOfWeekJoinDate: - title: Weekdays when these [[this.short-name]] joined + title: Weekdays when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -481,7 +481,7 @@ cards: group: ByTime x_label: Join date by day of the week - DayOfWeekJoinDate: - title: Weekdays when these [[this.short-name]] joined + title: Weekdays when [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -491,7 +491,7 @@ cards: group: ByTime x_label: Join date by day of the week - HourOfDayJoinDate: - title: Hours when these [[this.short-name]] joined + title: Hours when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -501,7 +501,7 @@ cards: group: ByTime x_label: Join date by hour of the day - HourOfDayJoinDate: - title: Hours when these [[this.short-name]] joined + title: Hours when [[this.short-name]] joined visualization: bar dimensions: - JoinTime: @@ -511,7 +511,7 @@ cards: group: ByTime x_label: Join date by hour of the day - DayOfMonthJoinDate: - title: Days of the month when these [[this.short-name]] joined + title: Days of the month when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -521,7 +521,7 @@ cards: group: ByTime x_label: Join date by day of the month - DayOfMonthJoinDate: - title: Days of the month when these [[this.short-name]] joined + title: Days of the month when [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -531,7 +531,7 @@ cards: group: ByTime x_label: Join date by day of the month - MonthOfYearJoinDate: - title: Months when these [[this.short-name]] joined + title: Months when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -541,7 +541,7 @@ cards: group: ByTime x_label: Join date by month of the year - MonthOfYearJoinDate: - title: Months when these [[this.short-name]] joined + title: Months when [[this.short-name]] joined visualization: bar dimensions: - JoinDate: @@ -551,7 +551,7 @@ cards: group: ByTime x_label: Join date by month of the year - QuerterOfYearJoinDate: - title: Quarters when these [[this.short-name]] joined + title: Quarters when [[this.short-name]] joined visualization: bar dimensions: - JoinTimestamp: @@ -561,7 +561,7 @@ cards: group: ByTime x_label: Join date by quarter of the year - QuerterOfYearJoinDate: - title: Quarters when these [[this.short-name]] joined + title: Quarters when [[this.short-name]] joined visualization: bar dimensions: - JoinDate: diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 5388ed90bbb46..9aeb11db39d48 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -129,7 +129,7 @@ [segment] (let [table (-> segment :table_id Table)] {:entity segment - :full-name (tru "{0} in {1} segment" (:display_name table) (:name segment)) + :full-name (tru "{0} in the {1} segment" (:display_name table) (:name segment)) :short-name (:display_name table) :source table :database (:db_id table) From a94b51db9603c6a86ec79f52c3c51cbf667d3a18 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 28 Jun 2018 21:31:33 +0200 Subject: [PATCH 025/116] explicitly test for show=1 --- test/metabase/automagic_dashboards/core_test.clj | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/metabase/automagic_dashboards/core_test.clj b/test/metabase/automagic_dashboards/core_test.clj index 42534e2572ebd..0ad20aefb94c1 100644 --- a/test/metabase/automagic_dashboards/core_test.clj +++ b/test/metabase/automagic_dashboards/core_test.clj @@ -112,9 +112,7 @@ ;; We want to both generate as many cards as we can to catch all aberrations, but also make sure ;; that size limiting works. (and (valid-dashboard? (automagic-analysis entity {:cell-query cell-query :show :all})) - (let [dashboard (automagic-analysis entity {:cell-query cell-query :show 1})] - (assert (->> dashboard :ordered_cards (keep :card) count (>= 1))) - (valid-dashboard? dashboard))))) + (valid-dashboard? (automagic-analysis entity {:cell-query cell-query :show 1}))))) (expect (with-rasta @@ -122,6 +120,15 @@ (->> (db/select Table :db_id (data/id)) (every? test-automagic-analysis))))) +(expect + (with-rasta + (with-dashboard-cleanup + (->> (automagic-analysis (Table (data/id :venues)) {:show 1}) + :ordered_cards + (filter :card) + count + (= 1))))) + (expect (with-rasta (with-dashboard-cleanup From 3daa3c451934ac91b3ecf3583f4074a6bbedb9d2 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Fri, 29 Jun 2018 16:33:53 +0200 Subject: [PATCH 026/116] Don't skip small categories in GenericTable --- resources/automagic_dashboards/table/GenericTable.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/resources/automagic_dashboards/table/GenericTable.yaml b/resources/automagic_dashboards/table/GenericTable.yaml index 5688f39d7df89..481e93447ad0d 100644 --- a/resources/automagic_dashboards/table/GenericTable.yaml +++ b/resources/automagic_dashboards/table/GenericTable.yaml @@ -20,10 +20,6 @@ dimensions: - Source: field_type: GenericTable.Source score: 100 - - GenericCategorySmall: - field_type: GenericTable.Category - score: 80 - max_cardinality: 5 - GenericCategoryMedium: field_type: GenericTable.Category score: 75 From 148aeff3149e62d58e07c62be8ec6237181948a1 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Fri, 29 Jun 2018 16:54:42 +0200 Subject: [PATCH 027/116] correctly flow query-filter to aggregate card constituents --- src/metabase/automagic_dashboards/core.clj | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 30ac07d8f1662..f3d889083d2bb 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -553,8 +553,7 @@ (let [order_by (build-order-by dimensions metrics order_by) metrics (map (partial get (:metrics context)) metrics) filters (cond-> (map (partial get (:filters context)) filters) - (:query-filter context) - (conj {:filter (:query-filter context)})) + (:query-filter context) (conj {:filter (:query-filter context)})) score (if query score (* (or (->> dimensions @@ -711,7 +710,7 @@ (-> rule (select-keys [:title :description :transient_title :groups]) (instantiate-metadata context {}) - (assoc :refinements (filters/inject-refinement (:query-filter root) (:cell-query root)))))) + (assoc :refinements (:query-filter context))))) (s/defn ^:private apply-rule [root, rule :- rules/Rule] @@ -869,8 +868,9 @@ (defn- decompose-question [root question opts] (map #(automagic-analysis % (assoc opts - :source (:source root) - :database (:database root))) + :source (:source root) + :query-filter (:query-filter root) + :database (:database root))) (concat (collect-metrics root question) (collect-breakout-fields root question)))) @@ -981,7 +981,6 @@ (merge (cond-> root cell-query (merge {:url cell-url :entity (:source root) - :query-filter (qp.util/get-in-normalized query [:dataset_query :query :filter]) :rules-prefix ["table"]})) opts)) (let [opts (assoc opts :show :all)] From 9dce3b78532fbf9c27a79cc339e742400585244a Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Fri, 29 Jun 2018 18:45:10 +0200 Subject: [PATCH 028/116] more robust save dashboard test --- test/metabase/models/dashboard_test.clj | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/metabase/models/dashboard_test.clj b/test/metabase/models/dashboard_test.clj index f3548eb6f6c6c..1c7338e6491f8 100644 --- a/test/metabase/models/dashboard_test.clj +++ b/test/metabase/models/dashboard_test.clj @@ -229,14 +229,15 @@ ;; test that we save a transient dashboard (expect - 8 (tu/with-model-cleanup ['Card 'Dashboard 'DashboardCard 'Collection] (binding [api/*current-user-id* (users/user->id :rasta) api/*current-user-permissions-set* (-> :rasta users/user->id user/permissions-set atom)] - (->> (magic/automagic-analysis (Table (id :venues)) {}) - save-transient-dashboard! - :id - (db/count 'DashboardCard :dashboard_id))))) + (let [dashboard (magic/automagic-analysis (Table (id :venues)) {})] + (->> dashboard + save-transient-dashboard! + :id + (db/count 'DashboardCard :dashboard_id) + (= (-> dashboard :ordered_cards count))))))) From 0676c858a1289e06f7896d9003526aff32f04e33 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Mon, 2 Jul 2018 19:18:53 +0200 Subject: [PATCH 029/116] Fix nested cards and improve filter merging --- src/metabase/automagic_dashboards/filters.clj | 19 +--- .../automagic_dashboards/populate.clj | 89 ++++++++++--------- 2 files changed, 51 insertions(+), 57 deletions(-) diff --git a/src/metabase/automagic_dashboards/filters.clj b/src/metabase/automagic_dashboards/filters.clj index 5f1a3c30ba879..85a8150ad560b 100644 --- a/src/metabase/automagic_dashboards/filters.clj +++ b/src/metabase/automagic_dashboards/filters.clj @@ -11,7 +11,7 @@ [(s/one (s/constrained su/KeywordOrString (comp #{:field-id :fk-> :field-literal} qp.util/normalize-token)) "head") - s/Any]) + (s/cond-pre s/Int su/KeywordOrString)]) (def ^:private ^{:arglists '([form])} field-reference? "Is given form an MBQL field reference?" @@ -25,7 +25,7 @@ (defmethod field-reference->id :field-id [[_ id]] (if (sequential? id) - (second id) + (field-reference->id id) id)) (defmethod field-reference->id :fk-> @@ -156,10 +156,10 @@ remove-unqualified (sort-by interestingness >) (take max-filters) - (map #(assoc % :fk-map (build-fk-map fks %))) (reduce (fn [dashboard candidate] - (let [filter-id (-> candidate hash str) + (let [filter-id (-> candidate ((juxt :id :name :unit)) hash str) + candidate (assoc candidate :fk-map (build-fk-map fks candidate)) dashcards (:ordered_cards dashboard) dashcards-new (map #(add-filter % filter-id candidate) dashcards)] ;; Only add filters that apply to all cards. @@ -174,17 +174,6 @@ dashboard))))) -(defn filter-referenced-fields - "Return a map of fields referenced in filter cluase." - [filter-clause] - (->> filter-clause - collect-field-references - (mapcat (fn [[_ & ids]] - (for [id ids] - [id (Field id)]))) - (into {}))) - - (defn- flatten-filter-clause [filter-clause] (when (not-empty filter-clause) diff --git a/src/metabase/automagic_dashboards/populate.clj b/src/metabase/automagic_dashboards/populate.clj index dadc286cd7b53..344447085171a 100644 --- a/src/metabase/automagic_dashboards/populate.clj +++ b/src/metabase/automagic_dashboards/populate.clj @@ -244,8 +244,6 @@ :else n) dashboard {:name title :transient_name (or transient_title title) - :transient_filters refinements - :param_fields (filters/filter-referenced-fields refinements) :description description :creator_id api/*current-user-id* :parameters []} @@ -276,45 +274,52 @@ line))) str/join)) +(defn- merge-filters + [ds] + (when (->> ds + (mapcat :ordered_cards) + (keep (comp :table_id :card)) + distinct + count + (= 1)) + [(->> ds (mapcat :parameters) distinct) + (->> ds + (mapcat :ordered_cards) + (mapcat :parameter_mappings) + (map #(dissoc % :card_id)) + distinct)])) + (defn merge-dashboards "Merge dashboards `ds` into dashboard `d`." - [d & ds] - (let [filter-targets (when (->> ds - (mapcat :ordered_cards) - (keep (comp :table_id :card)) - distinct - count - (= 1)) - (->> ds - (mapcat :ordered_cards) - (mapcat :parameter_mappings) - (mapcat (comp filters/collect-field-references :target)) - (map filters/field-reference->id) - distinct - (map Field)))] - (cond-> (reduce - (fn [target dashboard] - (let [offset (->> target - :ordered_cards - (map #(+ (:row %) (:sizeY %))) - (apply max -1) ; -1 so it neturalizes +1 for spacing if - ; the target dashboard is empty. - inc) - cards (->> dashboard - :ordered_cards - (map #(-> % - (update :row + offset group-heading-height) - (u/update-in-when [:visualization_settings :text] - downsize-titles) - (dissoc :parameter_mappings))))] - (-> target - (add-text-card {:width grid-width - :height group-heading-height - :text (format "# %s" (:name dashboard)) - :visualization-settings {:dashcard.background false - :text.align_vertical :bottom}} - [offset 0]) - (update :ordered_cards concat cards)))) - d - ds) - (not-empty filter-targets) (filters/add-filters filter-targets max-filters)))) + [& ds] + (let [[paramters parameter-mappings] (merge-filters ds)] + (reduce + (fn [target dashboard] + (let [offset (->> target + :ordered_cards + (map #(+ (:row %) (:sizeY %))) + (apply max -1) ; -1 so it neturalizes +1 for spacing if + ; the target dashboard is empty. + inc) + cards (->> dashboard + :ordered_cards + (map #(-> % + (update :row + offset group-heading-height) + (u/update-in-when [:visualization_settings :text] + downsize-titles) + (assoc :parameter_mappings + (when (:card_id %) + (for [mapping parameter-mappings] + (assoc mapping :card_id (:card_id %))))))))] + (-> target + (add-text-card {:width grid-width + :height group-heading-height + :text (format "# %s" (:name dashboard)) + :visualization-settings {:dashcard.background false + :text.align_vertical :bottom}} + [offset 0]) + (update :ordered_cards concat cards)))) + (-> ds + first + (assoc :parameters paramters)) + (rest ds)))) From 4abc441992a8b50457b0534698155c7b198ab0ed Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Mon, 2 Jul 2018 19:19:16 +0200 Subject: [PATCH 030/116] Fix a bunch of bugs --- src/metabase/automagic_dashboards/core.clj | 95 ++++++++++++++-------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index f3d889083d2bb..4a15fe1b52bdc 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -46,10 +46,10 @@ [root id-or-name] (if (->> root :source (instance? (type Table))) (Field id-or-name) - (let [field (->> root + (when-let [field (->> root :source :result_metadata - (some (comp #{id-or-name} :name)))] + (m/find-first (comp #{id-or-name} :name)))] (-> field (update :base_type keyword) (update :special_type keyword) @@ -70,21 +70,21 @@ :cum-count (tru "cumulative count") :cum-sum (tru "cumulative sum")}) -(defn- metric-name - [[op arg]] - (let [op (qp.util/normalize-token op)] - (if (= op :metric) - (-> arg Metric :name) - (op->name op)))) +(def ^:private ^{:arglists '([metric])} saved-metric? + (comp #{:metric} qp.util/normalize-token first)) -(defn- metric->description - [root metric] - (tru "{0} of {1}" (metric-name metric) (or (some->> metric - second - filters/field-reference->id - (->field root) - :display_name) - (source-name root)))) +(def ^:private ^{:arglists '([metric])} custom-expression? + (comp #{:named} qp.util/normalize-token first)) + +(def ^:private ^{:arglists '([metric])} adhoc-metric? + (complement (some-fn saved-metric? custom-expression?))) + +(defn- metric-name + [[op & args :as metric]] + (cond + (adhoc-metric? metric) (-> op qp.util/normalize-token op->name) + (saved-metric? metric) (-> args first Metric :name) + :else (second args))) (defn- join-enumeration [xs] @@ -92,11 +92,25 @@ (tru "{0} and {1}" (str/join ", " (butlast xs)) (last xs)) (first xs))) +(defn- metric->description + [root aggregation-clause] + (join-enumeration + (for [metric (if (sequential? (first aggregation-clause)) + aggregation-clause + [aggregation-clause])] + (if (adhoc-metric? metric) + (tru "{0} of {1}" (metric-name metric) (or (some->> metric + second + filters/field-reference->id + (->field root) + :display_name) + (source-name root))) + (metric-name metric))))) + (defn- question-description [root question] (let [aggregations (->> (qp.util/get-in-normalized question [:dataset_query :query :aggregation]) - (map (partial metric->description root)) - join-enumeration) + (metric->description root)) dimensions (->> (qp.util/get-in-normalized question [:dataset_query :query :breakout]) (mapcat filters/collect-field-references) (map (comp :display_name @@ -135,7 +149,7 @@ :short-name (:display_name table) :source table :database (:db_id table) - :query-filter (-> segment :definition :filter) + :query-filter [:SEGMENT (u/get-id segment)] :url (format "%ssegment/%s" public-endpoint (u/get-id segment)) :rules-prefix ["table"]})) @@ -143,7 +157,9 @@ [metric] (let [table (-> metric :table_id Table)] {:entity metric - :full-name (tru "{0} metric" (:name metric)) + :full-name (if (:id metric) + (tru "{0} metric" (:name metric)) + (:name metric)) :short-name (:name metric) :source table :database (:db_id table) @@ -677,7 +693,7 @@ (update :special_type keyword) field/map->FieldInstance (classify/run-classifiers {}) - (map #(assoc % :engine engine))))) + (assoc :engine engine)))) constantly))] (as-> {:source (assoc source :fields (table->fields source)) :root root @@ -709,8 +725,7 @@ ([root rule context] (-> rule (select-keys [:title :description :transient_title :groups]) - (instantiate-metadata context {}) - (assoc :refinements (:query-filter context))))) + (instantiate-metadata context {})))) (s/defn ^:private apply-rule [root, rule :- rules/Rule] @@ -791,6 +806,17 @@ {:related (related-entities n-related-entities root) :drilldown-fields (take (- max-related n-related-entities) drilldown-fields)})))) +(defn- filter-referenced-fields + "Return a map of fields referenced in filter cluase." + [root filter-clause] + (->> filter-clause + filters/collect-field-references + (mapcat (fn [[_ & ids]] + (for [id ids] + [id (->field root id)]))) + (remove (comp nil? second)) + (into {}))) + (defn- automagic-dashboard "Create dashboards for table `root` using the best matching heuristics." [{:keys [rule show rules-prefix full-name] :as root}] @@ -815,10 +841,12 @@ (-> context :filters u/pprint-to-str)) (-> dashboard (populate/create-dashboard show) - (assoc :related (related context rule) - :more (when (and (not= show :all) - (-> dashboard :cards count (> show))) - (format "%s#show=all" (:url root)))))) + (assoc :related (related context rule) + :more (when (and (not= show :all) + (-> dashboard :cards count (> show))) + (format "%s#show=all" (:url root))) + :transient_filters (:query-filter context) + :param_fields (->> context :query-filter (filter-referenced-fields root))))) (throw (ex-info (trs "Can''t create dashboard for {0}" full-name) {:root root :available-rules (map :rule (or (some-> rule rules/get-rule vector) @@ -850,11 +878,11 @@ qp.util/normalize-token (= :metric)) (-> aggregation-clause second Metric) - (let [metric (metric/map->MetricInstance - {:definition {:aggregation [aggregation-clause] - :source_table (:table_id question)} - :table_id (:table_id question)})] - (assoc metric :name (metric->description root aggregation-clause))))) + (let [table-id ((some-fn :table-id :table_id) question)] + (metric/map->MetricInstance {:definition {:aggregation [aggregation-clause] + :source_table table-id} + :name (metric->description root aggregation-clause) + :table_id table-id})))) (qp.util/get-in-normalized question [:dataset_query :query :aggregation]))) (defn- collect-breakout-fields @@ -943,8 +971,7 @@ (defn- cell-title [root cell-query] (str/join " " [(->> (qp.util/get-in-normalized (-> root :entity) [:dataset_query :query :aggregation]) - (map (partial metric->description root)) - join-enumeration) + (metric->description root)) (tru "where {0}" (humanize-filter-value root cell-query))])) (defmethod automagic-analysis (type Card) From cc59e2979f3b9b6d260a208845b5f5056075dfbc Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Mon, 2 Jul 2018 19:27:57 +0200 Subject: [PATCH 031/116] remove unneeded require --- src/metabase/automagic_dashboards/populate.clj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/metabase/automagic_dashboards/populate.clj b/src/metabase/automagic_dashboards/populate.clj index 344447085171a..60fd4bce4fa89 100644 --- a/src/metabase/automagic_dashboards/populate.clj +++ b/src/metabase/automagic_dashboards/populate.clj @@ -4,9 +4,7 @@ [clojure.tools.logging :as log] [metabase.api.common :as api] [metabase.automagic-dashboards.filters :as filters] - [metabase.models - [card :as card] - [field :refer [Field]]] + [metabase.models.card :as card] [metabase.query-processor.util :as qp.util] [metabase.util :as u] [puppetlabs.i18n.core :as i18n :refer [trs]] From c12c5e8a45a38bbd843de379486a1bea7f893fbf Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Mon, 2 Jul 2018 23:55:35 +0200 Subject: [PATCH 032/116] Flatten filters where possible --- src/metabase/automagic_dashboards/core.clj | 6 +++--- src/metabase/automagic_dashboards/filters.clj | 2 +- src/metabase/query_processor/middleware/expand_macros.clj | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 4a15fe1b52bdc..fe7b536b068cd 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -480,9 +480,9 @@ (-> context :source u/get-id) (->> context :source u/get-id (str "card__")))} (not-empty filters) - (assoc :filter (transduce (map :filter) - merge-filter-clauses - filters)) + (assoc :filter (->> filters + (map :filter) + (apply merge-filter-clauses))) (not-empty dimensions) (assoc :breakout dimensions) diff --git a/src/metabase/automagic_dashboards/filters.clj b/src/metabase/automagic_dashboards/filters.clj index 85a8150ad560b..a560f807175fb 100644 --- a/src/metabase/automagic_dashboards/filters.clj +++ b/src/metabase/automagic_dashboards/filters.clj @@ -199,4 +199,4 @@ (->> filter-clause flatten-filter-clause (remove (comp in-refinement? collect-field-references)) - (reduce merge-filter-clauses refinement)))) + (apply merge-filter-clauses refinement)))) diff --git a/src/metabase/query_processor/middleware/expand_macros.clj b/src/metabase/query_processor/middleware/expand_macros.clj index b40b54d579a4d..59a08c72830e9 100644 --- a/src/metabase/query_processor/middleware/expand_macros.clj +++ b/src/metabase/query_processor/middleware/expand_macros.clj @@ -116,12 +116,12 @@ "Merge filter clauses." ([] []) ([clause] clause) - ([base-clause additional-clauses] + ([base-clause & additional-clauses] (cond (and (seq base-clause) - (seq additional-clauses)) [:and base-clause additional-clauses] + (seq additional-clauses)) (apply vector :and base-clause additional-clauses) (seq base-clause) base-clause - (seq additional-clauses) additional-clauses + (seq additional-clauses) (apply merge-filter-clauses additional-clauses) :else []))) (defn- add-metrics-filter-clauses From cc54cd051b1e6bc3302c4c6df41b56ee170c9fa4 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Wed, 4 Jul 2018 20:11:22 +0200 Subject: [PATCH 033/116] Don't show superfluous datetime components --- src/metabase/automagic_dashboards/core.clj | 42 ++++++++++++------- .../middleware/expand_macros.clj | 13 +++--- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index fe7b536b068cd..2949c31ff2735 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -32,6 +32,7 @@ [metabase.related :as related] [metabase.sync.analyze.classify :as classify] [metabase.util :as u] + [metabase.util.date :as date] [puppetlabs.i18n.core :as i18n :refer [tru trs]] [ring.util.codec :as codec] [schema.core :as s] @@ -902,15 +903,23 @@ (concat (collect-metrics root question) (collect-breakout-fields root question)))) -(def ^:private date-formatter (t.format/formatter "MMMM d, YYYY")) -(def ^:private datetime-formatter (t.format/formatter "EEEE, MMMM d, YYYY h:mm a")) - (defn- humanize-datetime - [dt] - (t.format/unparse (if (str/index-of dt "T") - datetime-formatter - date-formatter) - (t.format/parse dt))) + [dt unit] + (let [dt (t.format/parse dt) + unparse-with-formatter (fn [formatter dt] + (t.format/unparse (t.format/formatter formatter) dt))] + (case unit + :minute (tru "ot {0}" (unparse-with-formatter "EEEE, MMMM d, YYYY h:mm a" dt)) + :hour (tru "at {0}" (unparse-with-formatter "EEEE, MMMM d, YYYY h a" dt)) + :day (tru "on {0}" (unparse-with-formatter "MMMM d, YYYY" dt)) + :month (tru "in {0}" (unparse-with-formatter "MMMM YYYY" dt)) + :quarter (tru "in Q{0} {1}" + (date/date-extract :quarter-of-year dt) + (->> dt (date/date-extract :year) str)) + :year (date/date-extract :year dt) + :day-of-week (tru "on a {}" (unparse-with-formatter "EEEE" dt)) + (:minute-of-hour :hour-of-day ::day-of-month :week-of-year :month-of-year :quarter-of-year) + (date/date-extract unit dt)))) (defn- field-reference->field [root field-reference] @@ -928,13 +937,13 @@ humanize-filter-value (fn [_ [op & args]] (qp.util/normalize-token op))) -(def ^:private unit-name (comp {:minute-of-hour "minute of hour" - :hour-of-day "hour of day" - :day-of-week "day of week" +(def ^:private unit-name (comp {:minute-of-hour "minute" + :hour-of-day "hour" + :day-of-week "" :day-of-month "day of month" - :week-of-year "week of year" - :month-of-year "month of year" - :quarter-of-year "quarter of year"} + :week-of-year "week of" + :month-of-year "month of" + :quarter-of-year "quarter of"} qp.util/normalize-token)) (defn- field-name @@ -948,8 +957,9 @@ [root [_ field-reference value]] (let [field (field-reference->field root field-reference) field-name (field-name field)] - (if (filters/datetime? field) - (tru "{0} is on {1}" field-name (humanize-datetime value)) + (if (or (filters/datetime? field) + (filters/periodic-datetime? field)) + (tru "{0} is {1}" field-name (humanize-datetime value (:unit field))) (tru "{0} is {1}" field-name value)))) (defmethod humanize-filter-value :between diff --git a/src/metabase/query_processor/middleware/expand_macros.clj b/src/metabase/query_processor/middleware/expand_macros.clj index 59a08c72830e9..5e778463d6fd1 100644 --- a/src/metabase/query_processor/middleware/expand_macros.clj +++ b/src/metabase/query_processor/middleware/expand_macros.clj @@ -117,12 +117,13 @@ ([] []) ([clause] clause) ([base-clause & additional-clauses] - (cond - (and (seq base-clause) - (seq additional-clauses)) (apply vector :and base-clause additional-clauses) - (seq base-clause) base-clause - (seq additional-clauses) (apply merge-filter-clauses additional-clauses) - :else []))) + (let [additional-clauses (filter seq additional-clauses)] + (cond + (and (seq base-clause) + (seq additional-clauses)) (apply vector :and base-clause additional-clauses) + (seq base-clause) base-clause + (seq additional-clauses) (apply merge-filter-clauses additional-clauses) + :else [])))) (defn- add-metrics-filter-clauses "Add any FILTER-CLAUSES to the QUERY-DICT. If query has existing filter clauses, the new ones are From 70e18fb326f07c89802620b7f1bd89274f8f7676 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Wed, 4 Jul 2018 23:23:42 +0100 Subject: [PATCH 034/116] update tests --- test/metabase/automagic_dashboards/filters_test.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/metabase/automagic_dashboards/filters_test.clj b/test/metabase/automagic_dashboards/filters_test.clj index 28f3e6584d083..0c64afef3910c 100644 --- a/test/metabase/automagic_dashboards/filters_test.clj +++ b/test/metabase/automagic_dashboards/filters_test.clj @@ -12,7 +12,7 @@ ;; If there's no overlap between filter clauses, just merge using `:and`. (expect - [:and [:and [:and [:= [:field-id 3] 42] [:= [:fk-> 1 9] "foo"]] [:> [:field-id 2] 10]] [:< [:field-id 2] 100]] + [:and [:= [:field-id 3] 42] [:= [:fk-> 1 9] "foo"] [:> [:field-id 2] 10] [:< [:field-id 2] 100]] (inject-refinement [:and [:= [:fk-> 1 9] "foo"] [:and [:> [:field-id 2] 10] [:< [:field-id 2] 100]]] From fea5b2f426583d8f50c7f8800ecf5f711614bd77 Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Thu, 5 Jul 2018 11:25:36 -0700 Subject: [PATCH 035/116] fix tiny typo --- src/metabase/automagic_dashboards/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 2949c31ff2735..e7314f478f1a4 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -909,7 +909,7 @@ unparse-with-formatter (fn [formatter dt] (t.format/unparse (t.format/formatter formatter) dt))] (case unit - :minute (tru "ot {0}" (unparse-with-formatter "EEEE, MMMM d, YYYY h:mm a" dt)) + :minute (tru "on {0}" (unparse-with-formatter "EEEE, MMMM d, YYYY, h:mm a" dt)) :hour (tru "at {0}" (unparse-with-formatter "EEEE, MMMM d, YYYY h a" dt)) :day (tru "on {0}" (unparse-with-formatter "MMMM d, YYYY" dt)) :month (tru "in {0}" (unparse-with-formatter "MMMM YYYY" dt)) From e07a43e0d4e143a5af0e46b7a22f5e849ff6b715 Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Thu, 5 Jul 2018 11:49:00 -0700 Subject: [PATCH 036/116] reorder minute and hour titles, remove comma from year titles --- src/metabase/automagic_dashboards/core.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index e7314f478f1a4..ba4ec83d2c2c6 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -909,14 +909,14 @@ unparse-with-formatter (fn [formatter dt] (t.format/unparse (t.format/formatter formatter) dt))] (case unit - :minute (tru "on {0}" (unparse-with-formatter "EEEE, MMMM d, YYYY, h:mm a" dt)) - :hour (tru "at {0}" (unparse-with-formatter "EEEE, MMMM d, YYYY h a" dt)) + :minute (tru "at {0}" (unparse-with-formatter "h:mm a, MMMM d, YYYY" dt)) + :hour (tru "at {0}" (unparse-with-formatter "h a, MMMM d, YYYY" dt)) :day (tru "on {0}" (unparse-with-formatter "MMMM d, YYYY" dt)) - :month (tru "in {0}" (unparse-with-formatter "MMMM YYYY" dt)) - :quarter (tru "in Q{0} {1}" + :month (tru "in {0}" (unparse-with-formatter "MMMM, YYYY" dt)) + :quarter (tru "in Q{0}, {1}" (date/date-extract :quarter-of-year dt) (->> dt (date/date-extract :year) str)) - :year (date/date-extract :year dt) + :year (tru "{0}" (unparse-with-formatter "YYYY" dt)) :day-of-week (tru "on a {}" (unparse-with-formatter "EEEE" dt)) (:minute-of-hour :hour-of-day ::day-of-month :week-of-year :month-of-year :quarter-of-year) (date/date-extract unit dt)))) From b4f74a84409c7513f58b559f3b27436f259ee258 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Thu, 5 Jul 2018 20:55:53 +0100 Subject: [PATCH 037/116] Humanize periodical datetime components nicer --- src/metabase/automagic_dashboards/core.clj | 43 +++++++++++++++------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index ba4ec83d2c2c6..80193df6eb295 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -903,23 +903,38 @@ (concat (collect-metrics root question) (collect-breakout-fields root question)))) +(defn- pluralize + [x] + (case (mod x 10) + 1 (tru "{0}st" x) + 2 (tru "{0}nd" x) + 3 (tru "{0}rd" x) + (tru "{0}th" x))) + (defn- humanize-datetime [dt unit] (let [dt (t.format/parse dt) unparse-with-formatter (fn [formatter dt] (t.format/unparse (t.format/formatter formatter) dt))] (case unit - :minute (tru "at {0}" (unparse-with-formatter "h:mm a, MMMM d, YYYY" dt)) - :hour (tru "at {0}" (unparse-with-formatter "h a, MMMM d, YYYY" dt)) - :day (tru "on {0}" (unparse-with-formatter "MMMM d, YYYY" dt)) - :month (tru "in {0}" (unparse-with-formatter "MMMM, YYYY" dt)) - :quarter (tru "in Q{0}, {1}" - (date/date-extract :quarter-of-year dt) - (->> dt (date/date-extract :year) str)) - :year (tru "{0}" (unparse-with-formatter "YYYY" dt)) - :day-of-week (tru "on a {}" (unparse-with-formatter "EEEE" dt)) - (:minute-of-hour :hour-of-day ::day-of-month :week-of-year :month-of-year :quarter-of-year) - (date/date-extract unit dt)))) + :minute (tru "at {0}" (unparse-with-formatter "h:mm a, MMMM d, YYYY" dt)) + :hour (tru "at {0}" (unparse-with-formatter "h a, MMMM d, YYYY" dt)) + :day (tru "on {0}" (unparse-with-formatter "MMMM d, YYYY" dt)) + :week (tru "in {0} week - {1}" + (->> dt (date/date-extract :week-of-year) pluralize) + (->> dt (date/date-extract :year) str)) + :month (tru "in {0}" (unparse-with-formatter "MMMM, YYYY" dt)) + :quarter (tru "in Q{0} - {1}" + (date/date-extract :quarter-of-year dt) + (->> dt (date/date-extract :year) str)) + :year (unparse-with-formatter "YYYY" dt) + :day-of-week (tru "on a {0}" (unparse-with-formatter "EEEE" dt)) + :hour-of-day (tru "at {0}" (unparse-with-formatter "h a" dt)) + :month-of-year (unparse-with-formatter "MMMM" dt) + :quarter-of-year (tru "Q{0}" (date/date-extract :quarter-of-year dt)) + (:minute-of-hour + :day-of-month + :week-of-year) (date/date-extract unit dt)))) (defn- field-reference->field [root field-reference] @@ -941,9 +956,9 @@ :hour-of-day "hour" :day-of-week "" :day-of-month "day of month" - :week-of-year "week of" - :month-of-year "month of" - :quarter-of-year "quarter of"} + :week-of-year "week" + :month-of-year "month" + :quarter-of-year "quarter"} qp.util/normalize-token)) (defn- field-name From 0eef85692a8f8193e5de6c0559af64e361034fed Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Fri, 6 Jul 2018 15:48:45 +0100 Subject: [PATCH 038/116] Fix related --- src/metabase/automagic_dashboards/core.clj | 10 +++++----- src/metabase/related.clj | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 80193df6eb295..b5d139395712a 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -206,7 +206,7 @@ source-question (assoc :entity_type :entity/GenericTable)) (native-query? card) (-> card (assoc :entity_type :entity/GenericTable)) - :else (-> card ((some-fn :table_id :table-id)) Table))) + :else (-> card (qp.util/get-normalized :table-id) Table))) (defmethod ->root (type Card) [card] @@ -311,9 +311,9 @@ (defmethod ->reference :default [_ form] - (or (cond-> form - (map? form) ((some-fn :full-name :name) form)) - form)) + (cond + (map? form) ((some-fn :display_name :name) form) + :else form)) (defn- field-isa? [{:keys [base_type special_type]} t] @@ -879,7 +879,7 @@ qp.util/normalize-token (= :metric)) (-> aggregation-clause second Metric) - (let [table-id ((some-fn :table-id :table_id) question)] + (let [table-id (qp.util/get-normalized question :table-id)] (metric/map->MetricInstance {:definition {:aggregation [aggregation-clause] :source_table table-id} :name (metric->description root aggregation-clause) diff --git a/src/metabase/related.clj b/src/metabase/related.clj index 6d66efbe25352..668466e47de52 100644 --- a/src/metabase/related.clj +++ b/src/metabase/related.clj @@ -146,7 +146,7 @@ (defn- similar-questions [card] (->> (db/select Card - :table_id (:table_id card) + :table_id (qp.util/get-normalized card :table-id) :archived false) filter-visible (rank-by-similarity card) @@ -155,7 +155,7 @@ (defn- canonical-metric [card] (->> (db/select Metric - :table_id (:table_id card) + :table_id (qp.util/get-normalized card :table-id) :archived false) filter-visible (m/find-first (comp #{(-> card :dataset_query :query :aggregation)} @@ -206,7 +206,7 @@ (defmethod related (type Card) [card] - (let [table (Table (:table_id card)) + (let [table (Table (qp.util/get-normalized card :table-id)) similar-questions (similar-questions card)] {:table table :metrics (->> table From 6b2c2b2d1722e5d17287e6f3da6870aa24cbf075 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Fri, 6 Jul 2018 17:49:06 +0100 Subject: [PATCH 039/116] fix regression --- src/metabase/automagic_dashboards/core.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index b5d139395712a..c2275595f4400 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -311,9 +311,9 @@ (defmethod ->reference :default [_ form] - (cond - (map? form) ((some-fn :display_name :name) form) - :else form)) + (or (cond-> form + (map? form) ((some-fn :full-name :name) form)) + form)) (defn- field-isa? [{:keys [base_type special_type]} t] From e5dd6077b04c9e727071469a898467393c5fa21b Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Sat, 7 Jul 2018 20:36:23 +0100 Subject: [PATCH 040/116] Properly handle timezone --- src/metabase/automagic_dashboards/core.clj | 21 +++++++++++++-------- src/metabase/util/date.clj | 3 ++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index c2275595f4400..7eea839589175 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -913,28 +913,33 @@ (defn- humanize-datetime [dt unit] - (let [dt (t.format/parse dt) + (let [dt (date/str->date-time dt) + tz (-> date/jvm-timezone deref .getID) unparse-with-formatter (fn [formatter dt] - (t.format/unparse (t.format/formatter formatter) dt))] + (t.format/unparse + (t.format/with-zone + (t.format/formatter formatter) + (t/time-zone-for-id tz)) + dt))] (case unit :minute (tru "at {0}" (unparse-with-formatter "h:mm a, MMMM d, YYYY" dt)) :hour (tru "at {0}" (unparse-with-formatter "h a, MMMM d, YYYY" dt)) :day (tru "on {0}" (unparse-with-formatter "MMMM d, YYYY" dt)) :week (tru "in {0} week - {1}" - (->> dt (date/date-extract :week-of-year) pluralize) - (->> dt (date/date-extract :year) str)) + (->> dt (date/date-extract :week-of-year tz) pluralize) + (->> dt (date/date-extract :year tz) str)) :month (tru "in {0}" (unparse-with-formatter "MMMM, YYYY" dt)) :quarter (tru "in Q{0} - {1}" - (date/date-extract :quarter-of-year dt) - (->> dt (date/date-extract :year) str)) + (date/date-extract :quarter-of-year tz dt) + (->> dt (date/date-extract :year tz) str)) :year (unparse-with-formatter "YYYY" dt) :day-of-week (tru "on a {0}" (unparse-with-formatter "EEEE" dt)) :hour-of-day (tru "at {0}" (unparse-with-formatter "h a" dt)) :month-of-year (unparse-with-formatter "MMMM" dt) - :quarter-of-year (tru "Q{0}" (date/date-extract :quarter-of-year dt)) + :quarter-of-year (tru "Q{0}" (date/date-extract :quarter-of-year tz dt)) (:minute-of-hour :day-of-month - :week-of-year) (date/date-extract unit dt)))) + :week-of-year) (date/date-extract unit tz dt)))) (defn- field-reference->field [root field-reference] diff --git a/src/metabase/util/date.clj b/src/metabase/util/date.clj index 23b2775a420bc..3c353f88535bc 100644 --- a/src/metabase/util/date.clj +++ b/src/metabase/util/date.clj @@ -40,7 +40,8 @@ "UTC TimeZone" (coerce-to-timezone "UTC")) -(def ^:private jvm-timezone +(def jvm-timezone + "Machine time zone" (delay (coerce-to-timezone (System/getProperty "user.timezone")))) (defn- warn-on-timezone-conflict From 432929ecc75ea5fabdc84f2764fc616c5b12b05e Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Sat, 7 Jul 2018 23:02:51 +0100 Subject: [PATCH 041/116] Add type hint --- src/metabase/automagic_dashboards/core.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 7eea839589175..5789efae3d1f8 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -36,7 +36,8 @@ [puppetlabs.i18n.core :as i18n :refer [tru trs]] [ring.util.codec :as codec] [schema.core :as s] - [toucan.db :as db])) + [toucan.db :as db]) + (:import java.util.TimeZone)) (def ^:private public-endpoint "/auto/dashboard/") @@ -914,7 +915,7 @@ (defn- humanize-datetime [dt unit] (let [dt (date/str->date-time dt) - tz (-> date/jvm-timezone deref .getID) + tz (-> date/jvm-timezone deref ^TimeZone .getID) unparse-with-formatter (fn [formatter dt] (t.format/unparse (t.format/with-zone From d4ce7b8f70fc2181a71bd322aa78c52254034cd4 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Sun, 8 Jul 2018 13:34:36 +0200 Subject: [PATCH 042/116] fix wrong arg order. ADD TESTS for datetime humanization --- src/metabase/automagic_dashboards/core.clj | 18 +++--- .../automagic_dashboards/core_test.clj | 59 ++++++++++++++++++- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 5789efae3d1f8..f33fe0b6cb4e5 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -918,29 +918,27 @@ tz (-> date/jvm-timezone deref ^TimeZone .getID) unparse-with-formatter (fn [formatter dt] (t.format/unparse - (t.format/with-zone - (t.format/formatter formatter) - (t/time-zone-for-id tz)) + (t.format/formatter formatter (t/time-zone-for-id tz)) dt))] (case unit :minute (tru "at {0}" (unparse-with-formatter "h:mm a, MMMM d, YYYY" dt)) :hour (tru "at {0}" (unparse-with-formatter "h a, MMMM d, YYYY" dt)) :day (tru "on {0}" (unparse-with-formatter "MMMM d, YYYY" dt)) :week (tru "in {0} week - {1}" - (->> dt (date/date-extract :week-of-year tz) pluralize) - (->> dt (date/date-extract :year tz) str)) - :month (tru "in {0}" (unparse-with-formatter "MMMM, YYYY" dt)) + (pluralize (date/date-extract :week-of-year dt tz)) + (str (date/date-extract :year dt tz))) + :month (tru "in {0}" (unparse-with-formatter "MMMM YYYY" dt)) :quarter (tru "in Q{0} - {1}" - (date/date-extract :quarter-of-year tz dt) - (->> dt (date/date-extract :year tz) str)) + (date/date-extract :quarter-of-year dt tz) + (str (date/date-extract :year dt tz))) :year (unparse-with-formatter "YYYY" dt) :day-of-week (tru "on a {0}" (unparse-with-formatter "EEEE" dt)) :hour-of-day (tru "at {0}" (unparse-with-formatter "h a" dt)) :month-of-year (unparse-with-formatter "MMMM" dt) - :quarter-of-year (tru "Q{0}" (date/date-extract :quarter-of-year tz dt)) + :quarter-of-year (tru "Q{0}" (date/date-extract :quarter-of-year dt tz)) (:minute-of-hour :day-of-month - :week-of-year) (date/date-extract unit tz dt)))) + :week-of-year) (date/date-extract unit dt tz)))) (defn- field-reference->field [root field-reference] diff --git a/test/metabase/automagic_dashboards/core_test.clj b/test/metabase/automagic_dashboards/core_test.clj index 0ad20aefb94c1..8b967614997f4 100644 --- a/test/metabase/automagic_dashboards/core_test.clj +++ b/test/metabase/automagic_dashboards/core_test.clj @@ -1,5 +1,8 @@ (ns metabase.automagic-dashboards.core-test - (:require [expectations :refer :all] + (:require [clj-time + [core :as t] + [format :as t.format]] + [expectations :refer :all] [metabase.api.common :as api] [metabase.automagic-dashboards [core :refer :all :as magic] @@ -19,6 +22,8 @@ [metabase.test.data :as data] [metabase.test.data.users :as test-users] [metabase.test.util :as tu] + [metabase.util.date :as date] + [puppetlabs.i18n.core :as i18n :refer [tru]] [toucan.db :as db] [toucan.util.test :as tt])) @@ -447,3 +452,55 @@ (#'magic/optimal-datetime-resolution {:fingerprint {:type {:type/DateTime {:earliest "2017-01-01T00:00:00" :latest "2017-01-01T00:02:00"}}}})) + + +;;; ------------------- Datetime humanization (for chart and dashboard titles) ------------------- + +(let [tz (-> date/jvm-timezone deref ^TimeZone .getID) + dt (t/from-time-zone (t/date-time 1990 9 9 12 30) + (t/time-zone-for-id tz)) + unparse-with-formatter (fn [formatter dt] + (t.format/unparse + (t.format/formatter formatter (t/time-zone-for-id tz)) dt))] + (expect + [(tru "at {0}" (unparse-with-formatter "h:mm a, MMMM d, YYYY" dt)) + (tru "at {0}" (unparse-with-formatter "h a, MMMM d, YYYY" dt)) + (tru "on {0}" (unparse-with-formatter "MMMM d, YYYY" dt)) + (tru "in {0} week - {1}" + (#'magic/pluralize (date/date-extract :week-of-year dt tz)) + (str (date/date-extract :year dt tz))) + (tru "in {0}" (unparse-with-formatter "MMMM YYYY" dt)) + (tru "in Q{0} - {1}" + (date/date-extract :quarter-of-year dt tz) + (str (date/date-extract :year dt tz))) + (unparse-with-formatter "YYYY" dt) + (tru "on a {0}" (unparse-with-formatter "EEEE" dt)) + (tru "at {0}" (unparse-with-formatter "h a" dt)) + (unparse-with-formatter "MMMM" dt) + (tru "Q{0}" (date/date-extract :quarter-of-year dt tz)) + (date/date-extract :minute-of-hour dt tz) + (date/date-extract :day-of-month dt tz) + (date/date-extract :week-of-year dt tz)] + (let [dt (t.format/unparse (t.format/formatters :date-hour-minute-second) dt)] + [(#'magic/humanize-datetime dt :minute) + (#'magic/humanize-datetime dt :hour) + (#'magic/humanize-datetime dt :day) + (#'magic/humanize-datetime dt :week) + (#'magic/humanize-datetime dt :month) + (#'magic/humanize-datetime dt :quarter) + (#'magic/humanize-datetime dt :year) + (#'magic/humanize-datetime dt :day-of-week) + (#'magic/humanize-datetime dt :hour-of-day) + (#'magic/humanize-datetime dt :month-of-year) + (#'magic/humanize-datetime dt :quarter-of-year) + (#'magic/humanize-datetime dt :minute-of-hour) + (#'magic/humanize-datetime dt :day-of-month) + (#'magic/humanize-datetime dt :week-of-year)]))) + +(expect + [(tru "{0}st" 1) + (tru "{0}nd" 22) + (tru "{0}rd" 303) + (tru "{0}th" 0) + (tru "{0}th" 8)] + (map #'magic/pluralize [1 22 303 0 8])) From e83cc4b2b5263c1366468991b65de8c19773a6ad Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Sun, 8 Jul 2018 17:44:39 +0200 Subject: [PATCH 043/116] Fix type hint --- src/metabase/automagic_dashboards/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index f33fe0b6cb4e5..cb4912a97a50f 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -915,7 +915,7 @@ (defn- humanize-datetime [dt unit] (let [dt (date/str->date-time dt) - tz (-> date/jvm-timezone deref ^TimeZone .getID) + tz (.getID ^TimeZone @date/jvm-timezone) unparse-with-formatter (fn [formatter dt] (t.format/unparse (t.format/formatter formatter (t/time-zone-for-id tz)) From 918f18a1ccd81faeebf1b8d787cc821800cff85e Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Mon, 9 Jul 2018 10:03:13 -0700 Subject: [PATCH 044/116] fix dangling in day-of-week xrays --- src/metabase/automagic_dashboards/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index cb4912a97a50f..70046a4bf001a 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -932,7 +932,7 @@ (date/date-extract :quarter-of-year dt tz) (str (date/date-extract :year dt tz))) :year (unparse-with-formatter "YYYY" dt) - :day-of-week (tru "on a {0}" (unparse-with-formatter "EEEE" dt)) + :day-of-week (tru "{0}" (unparse-with-formatter "EEEE" dt)) :hour-of-day (tru "at {0}" (unparse-with-formatter "h a" dt)) :month-of-year (unparse-with-formatter "MMMM" dt) :quarter-of-year (tru "Q{0}" (date/date-extract :quarter-of-year dt tz)) @@ -958,7 +958,7 @@ (def ^:private unit-name (comp {:minute-of-hour "minute" :hour-of-day "hour" - :day-of-week "" + :day-of-week "day of week" :day-of-month "day of month" :week-of-year "week" :month-of-year "month" From e87cf006ac1f8ab93557a74e63af1ca78abcdfc7 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Mon, 9 Jul 2018 20:56:09 +0200 Subject: [PATCH 045/116] Add day-of-year and test for all the available units --- src/metabase/automagic_dashboards/core.clj | 2 ++ test/metabase/automagic_dashboards/core_test.clj | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 70046a4bf001a..13a565bf856c3 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -938,6 +938,7 @@ :quarter-of-year (tru "Q{0}" (date/date-extract :quarter-of-year dt tz)) (:minute-of-hour :day-of-month + :day-of-year :week-of-year) (date/date-extract unit dt tz)))) (defn- field-reference->field @@ -960,6 +961,7 @@ :hour-of-day "hour" :day-of-week "day of week" :day-of-month "day of month" + :day-of-year "day of year" :week-of-year "week" :month-of-year "month" :quarter-of-year "quarter"} diff --git a/test/metabase/automagic_dashboards/core_test.clj b/test/metabase/automagic_dashboards/core_test.clj index 8b967614997f4..90b511dded0d7 100644 --- a/test/metabase/automagic_dashboards/core_test.clj +++ b/test/metabase/automagic_dashboards/core_test.clj @@ -504,3 +504,8 @@ (tru "{0}th" 0) (tru "{0}th" 8)] (map #'magic/pluralize [1 22 303 0 8])) + +;; Make sure we have handlers for all the units available +(expect + (every? (partial #'magic/humanize-datetime "1990-09-09T12:30:00") + (concat (var-get #'date/date-extract-units) (var-get #'date/date-trunc-units)))) From 2dfc38172e2dd18b421dc5206edf53df90705e01 Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Mon, 9 Jul 2018 23:12:37 +0200 Subject: [PATCH 046/116] Update test --- src/metabase/automagic_dashboards/core.clj | 2 +- test/metabase/automagic_dashboards/core_test.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index 13a565bf856c3..f3b5389095058 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -932,7 +932,7 @@ (date/date-extract :quarter-of-year dt tz) (str (date/date-extract :year dt tz))) :year (unparse-with-formatter "YYYY" dt) - :day-of-week (tru "{0}" (unparse-with-formatter "EEEE" dt)) + :day-of-week (unparse-with-formatter "EEEE" dt) :hour-of-day (tru "at {0}" (unparse-with-formatter "h a" dt)) :month-of-year (unparse-with-formatter "MMMM" dt) :quarter-of-year (tru "Q{0}" (date/date-extract :quarter-of-year dt tz)) diff --git a/test/metabase/automagic_dashboards/core_test.clj b/test/metabase/automagic_dashboards/core_test.clj index 90b511dded0d7..9e7c3f72a9a92 100644 --- a/test/metabase/automagic_dashboards/core_test.clj +++ b/test/metabase/automagic_dashboards/core_test.clj @@ -474,7 +474,7 @@ (date/date-extract :quarter-of-year dt tz) (str (date/date-extract :year dt tz))) (unparse-with-formatter "YYYY" dt) - (tru "on a {0}" (unparse-with-formatter "EEEE" dt)) + (unparse-with-formatter "EEEE" dt) (tru "at {0}" (unparse-with-formatter "h a" dt)) (unparse-with-formatter "MMMM" dt) (tru "Q{0}" (date/date-extract :quarter-of-year dt tz)) From 8a5b8bd8dbb4b305650f73b62b2b71dba87b9b46 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 9 Jul 2018 15:48:42 -0700 Subject: [PATCH 047/116] Sync table.columns to query's fields clause --- frontend/src/metabase/lib/dataset.js | 92 ++++++++++++++++++- frontend/src/metabase/lib/query/query.js | 20 +++- frontend/src/metabase/meta/types/Dataset.js | 2 + .../src/metabase/query_builder/actions.js | 16 +++- 4 files changed, 122 insertions(+), 8 deletions(-) diff --git a/frontend/src/metabase/lib/dataset.js b/frontend/src/metabase/lib/dataset.js index 02e819c304123..29208fc315f65 100644 --- a/frontend/src/metabase/lib/dataset.js +++ b/frontend/src/metabase/lib/dataset.js @@ -1,6 +1,21 @@ +/* @flow */ + import _ from "underscore"; -import type { Value, Column, DatasetData } from "metabase/meta/types/Dataset"; +import type { + Value, + Column, + ColumnName, + DatasetData, +} from "metabase/meta/types/Dataset"; +import type { Card, VisualizationSettings } from "metabase/meta/types/Card"; +import type { ConcreteField } from "metabase/meta/types/Query"; + +type ColumnSetting = { + name: ColumnName, + fieldRef?: ConcreteField, + enabled: boolean, +}; // Many aggregations result in [[null]] if there are no rows to aggregate after filters export const datasetContainsNoResults = (data: DatasetData): boolean => @@ -11,9 +26,80 @@ export const datasetContainsNoResults = (data: DatasetData): boolean => */ export const rangeForValue = ( value: Value, - column: Column, + column: ?Column, ): ?[number, number] => { - if (column && column.binning_info && column.binning_info.bin_width) { + if ( + typeof value === "number" && + column && + column.binning_info && + column.binning_info.bin_width + ) { return [value, value + column.binning_info.bin_width]; } }; + +/** + * Returns a MBQL field reference (ConcreteField) for a given result dataset column + * @param {Column} column Dataset result column + * @return {?ConcreteField} MBQL field reference + */ +export function fieldRefForColumn(column: Column): ?ConcreteField { + if (column.id != null) { + if (Array.isArray(column.id)) { + // $FlowFixMe: sometimes col.id is a field reference (e.x. nested queries), if so just return it + return column.id; + } else if (column.fk_field_id != null) { + return ["fk->", column.fk_field_id, column.id]; + } else { + return ["field-id", column.id]; + } + } else if (column["expression-name"] != null) { + return ["expression", column["expression-name"]]; + } else { + return null; + } +} + +/** + * Finds the column object from the dataset results for the given `table.columns` column setting + * @param {Column[]} columns Dataset results columns + * @param {ColumnSetting} columnSetting A "column setting" from the `table.columns` settings + * @return {?Column} A result column + */ +export function findColumnForColumnSetting( + columns: Column[], + columnSetting: ColumnSetting, +): ?Column { + return _.find( + columns, + column => + (columnSetting.fieldRef && + _.isEqual(columnSetting.fieldRef, fieldRefForColumn(column))) || + columnSetting.name === column.name, + ); +} + +/** + * Synchronizes the "table.columns" visualization setting to the structured + * query's `fields` + * @param {[type]} card Card to synchronize `fields`. Mutates value + * @param {[type]} cols Columns in last run results + */ +export function syncQueryFields(card: Card, cols: Column[]): void { + if ( + card.dataset_query.type === "query" && + card.visualization_settings["table.columns"] + ) { + const visibleColumns = card.visualization_settings["table.columns"] + .filter(columnSetting => columnSetting.enabled) + .map(columnSetting => findColumnForColumnSetting(cols, columnSetting)); + const fields = visibleColumns + .map(column => column && fieldRefForColumn(column)) + .filter(field => field); + if (!_.isEqual(card.dataset_query.query.fields, fields)) { + console.log("fields actual", card.dataset_query.query.fields); + console.log("fields expected", fields); + card.dataset_query.query.fields = fields; + } + } +} diff --git a/frontend/src/metabase/lib/query/query.js b/frontend/src/metabase/lib/query/query.js index ee356c62bb8fd..569a01173ba20 100644 --- a/frontend/src/metabase/lib/query/query.js +++ b/frontend/src/metabase/lib/query/query.js @@ -14,6 +14,7 @@ import type { ExpressionClause, ExpressionName, Expression, + FieldsClause, } from "metabase/meta/types/Query"; import type { TableMetadata } from "metabase/meta/types/Metadata"; @@ -94,6 +95,9 @@ export const removeOrderBy = (query: SQ, index: number) => export const clearOrderBy = (query: SQ) => setOrderByClause(query, O.clearOrderBy(query.order_by)); +// FIELD +export const clearFields = (query: SQ) => setFieldsClause(query, null); + // LIMIT export const getLimit = (query: SQ) => L.getLimit(query.limit); @@ -140,13 +144,14 @@ function setAggregationClause( ): SQ { let wasBareRows = A.isBareRows(query.aggregation); let isBareRows = A.isBareRows(aggregationClause); - // when switching to or from bare rows clear out any sorting clauses + // when switching to or from bare rows clear out any sorting and fields clauses if (isBareRows !== wasBareRows) { - clearOrderBy(query); + query = clearFields(query); + query = clearOrderBy(query); } // for bare rows we always clear out any dimensions because they don't make sense if (isBareRows) { - clearBreakouts(query); + query = clearBreakouts(query); } return setClause("aggregation", query, aggregationClause); } @@ -158,6 +163,8 @@ function setBreakoutClause(query: SQ, breakoutClause: ?BreakoutClause): SQ { query = removeOrderBy(query, index); } } + // clear fields when changing breakouts + query = clearFields(query); return setClause("breakout", query, breakoutClause); } function setFilterClause(query: SQ, filterClause: ?FilterClause): SQ { @@ -166,6 +173,9 @@ function setFilterClause(query: SQ, filterClause: ?FilterClause): SQ { function setOrderByClause(query: SQ, orderByClause: ?OrderByClause): SQ { return setClause("order_by", query, orderByClause); } +function setFieldsClause(query: SQ, fieldsClause: ?FieldsClause): SQ { + return setClause("fields", query, fieldsClause); +} function setLimitClause(query: SQ, limitClause: ?LimitClause): SQ { return setClause("limit", query, limitClause); } @@ -185,7 +195,9 @@ type FilterClauseName = | "breakout" | "order_by" | "limit" - | "expressions"; + | "expressions" + | "fields"; + function setClause(clauseName: FilterClauseName, query: SQ, clause: ?any): SQ { query = { ...query }; if (clause == null) { diff --git a/frontend/src/metabase/meta/types/Dataset.js b/frontend/src/metabase/meta/types/Dataset.js index 48c73d4d326c0..dc09def520d99 100644 --- a/frontend/src/metabase/meta/types/Dataset.js +++ b/frontend/src/metabase/meta/types/Dataset.js @@ -21,6 +21,8 @@ export type Column = { source?: "fields" | "aggregation" | "breakout", unit?: DatetimeUnit, binning_info?: BinningInfo, + fk_field_id?: FieldId, + "expression-name"?: any, }; export type Value = string | number | ISO8601Time | boolean | null | {}; diff --git a/frontend/src/metabase/query_builder/actions.js b/frontend/src/metabase/query_builder/actions.js index 78b4490c4f8f2..f5ec5f759c900 100644 --- a/frontend/src/metabase/query_builder/actions.js +++ b/frontend/src/metabase/query_builder/actions.js @@ -24,6 +24,7 @@ import { } from "metabase/lib/card"; import { formatSQL } from "metabase/lib/formatting"; import Query, { createQuery } from "metabase/lib/query"; +import { syncQueryFields } from "metabase/lib/dataset"; import { isPK } from "metabase/lib/types"; import Utils from "metabase/lib/utils"; import { getEngineNativeType, formatJsonQuery } from "metabase/lib/engine"; @@ -520,7 +521,13 @@ export const loadDatabaseFields = createThunkAction( }, ); -function updateVisualizationSettings(card, isEditing, display, vizSettings) { +function updateVisualizationSettings( + card, + isEditing, + display, + vizSettings, + result, +) { // don't need to store undefined vizSettings = Utils.copy(vizSettings); for (const name in vizSettings) { @@ -549,6 +556,10 @@ function updateVisualizationSettings(card, isEditing, display, vizSettings) { updatedCard.display = display; updatedCard.visualization_settings = vizSettings; + if (result && result.data && result.data.cols) { + syncQueryFields(updatedCard, result.data.cols); + } + return updatedCard; } @@ -569,6 +580,7 @@ export const setCardVisualization = createThunkAction( uiControls.isEditing, display, card.visualization_settings, + getFirstQueryResult(getState()), ); dispatch(updateUrl(updatedCard, { dirty: true })); return updatedCard; @@ -588,6 +600,7 @@ export const updateCardVisualizationSettings = createThunkAction( uiControls.isEditing, card.display, { ...card.visualization_settings, ...settings }, + getFirstQueryResult(getState()), ); dispatch(updateUrl(updatedCard, { dirty: true })); return updatedCard; @@ -607,6 +620,7 @@ export const replaceAllCardVisualizationSettings = createThunkAction( uiControls.isEditing, card.display, settings, + getFirstQueryResult(getState()), ); dispatch(updateUrl(updatedCard, { dirty: true })); return updatedCard; From 9056f92ef18dfd4a51fcf7d96370265d30fce97c Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 9 Jul 2018 17:59:03 -0700 Subject: [PATCH 048/116] First pass at UI for adding new fields in table chart settings --- .../lib/queries/StructuredQuery.js | 31 ++++++++------- frontend/src/metabase/lib/dataset.js | 12 ++++++ frontend/src/metabase/meta/types/Query.js | 2 +- .../src/metabase/query_builder/actions.js | 31 ++++++++++++++- .../components/VisualizationSettings.jsx | 17 +++++--- .../components/ChartSettings.jsx | 22 +++++++++-- .../settings/ChartSettingOrderedFields.jsx | 39 ++++++++++++++++++- .../visualizations/visualizations/Table.jsx | 1 + 8 files changed, 131 insertions(+), 24 deletions(-) diff --git a/frontend/src/metabase-lib/lib/queries/StructuredQuery.js b/frontend/src/metabase-lib/lib/queries/StructuredQuery.js index 6e251a792633f..ab7b4590d0593 100644 --- a/frontend/src/metabase-lib/lib/queries/StructuredQuery.js +++ b/frontend/src/metabase-lib/lib/queries/StructuredQuery.js @@ -635,12 +635,12 @@ export default class StructuredQuery extends AtomicQuery { return this._updateQuery(Q.removeExpression, arguments); } - // FIELD OPTIONS + // DIMENSION OPTIONS // TODO Atte Keinänen 6/18/17: Refactor to dimensionOptions which takes a dimensionFilter // See aggregationFieldOptions for an explanation why that covers more use cases - fieldOptions(fieldFilter = () => true): DimensionOptions { - const fieldOptions = { + dimensionOptions(dimensionFilter = () => true): DimensionOptions { + const dimensionOptions = { count: 0, fks: [], dimensions: [], @@ -648,11 +648,6 @@ export default class StructuredQuery extends AtomicQuery { const table = this.tableMetadata(); if (table) { - const dimensionFilter = dimension => { - const field = dimension.field && dimension.field(); - return !field || (field.isDimension() && fieldFilter(field)); - }; - const dimensionIsFKReference = dimension => dimension.field && dimension.field() && dimension.field().isFK(); @@ -660,8 +655,8 @@ export default class StructuredQuery extends AtomicQuery { // .filter(d => !dimensionIsFKReference(d)); for (const dimension of filteredNonFKDimensions) { - fieldOptions.count++; - fieldOptions.dimensions.push(dimension); + dimensionOptions.count++; + dimensionOptions.dimensions.push(dimension); } const fkDimensions = this.dimensions().filter(dimensionIsFKReference); @@ -671,8 +666,8 @@ export default class StructuredQuery extends AtomicQuery { .filter(dimensionFilter); if (fkDimensions.length > 0) { - fieldOptions.count += fkDimensions.length; - fieldOptions.fks.push({ + dimensionOptions.count += fkDimensions.length; + dimensionOptions.fks.push({ field: dimension.field(), dimension: dimension, dimensions: fkDimensions, @@ -681,7 +676,17 @@ export default class StructuredQuery extends AtomicQuery { } } - return fieldOptions; + return dimensionOptions; + } + + // FIELD OPTIONS + + fieldOptions(fieldFilter = () => true) { + const dimensionFilter = dimension => { + const field = dimension.field && dimension.field(); + return !field || (field.isDimension() && fieldFilter(field)); + }; + return this.dimensionOptions(dimensionFilter); } // DIMENSIONS diff --git a/frontend/src/metabase/lib/dataset.js b/frontend/src/metabase/lib/dataset.js index 29208fc315f65..5600ed6d3605a 100644 --- a/frontend/src/metabase/lib/dataset.js +++ b/frontend/src/metabase/lib/dataset.js @@ -1,6 +1,7 @@ /* @flow */ import _ from "underscore"; +import * as Q from "metabase/lib/query/query"; import type { Value, @@ -103,3 +104,14 @@ export function syncQueryFields(card: Card, cols: Column[]): void { } } } + +export function getExistingFields(card: Card, cols: Column[]): ConcreteField[] { + const query = card.dataset_query.query; + if (query.fields && query.fields > 0) { + return query.fields; + } else if (Q.isBareRows(query)) { + return cols.map(col => fieldRefForColumn(col)).filter(id => id != null); + } else { + return []; + } +} diff --git a/frontend/src/metabase/meta/types/Query.js b/frontend/src/metabase/meta/types/Query.js index abf0897ab6d30..e73aa57328916 100644 --- a/frontend/src/metabase/meta/types/Query.js +++ b/frontend/src/metabase/meta/types/Query.js @@ -271,4 +271,4 @@ export type Expression = [ export type ExpressionOperator = "+" | "-" | "*" | "/"; export type ExpressionOperand = ConcreteField | NumericLiteral | Expression; -export type FieldsClause = Field[]; +export type FieldsClause = ConcreteField[]; diff --git a/frontend/src/metabase/query_builder/actions.js b/frontend/src/metabase/query_builder/actions.js index f5ec5f759c900..cf705d96e5a72 100644 --- a/frontend/src/metabase/query_builder/actions.js +++ b/frontend/src/metabase/query_builder/actions.js @@ -24,7 +24,7 @@ import { } from "metabase/lib/card"; import { formatSQL } from "metabase/lib/formatting"; import Query, { createQuery } from "metabase/lib/query"; -import { syncQueryFields } from "metabase/lib/dataset"; +import { syncQueryFields, getExistingFields } from "metabase/lib/dataset"; import { isPK } from "metabase/lib/types"; import Utils from "metabase/lib/utils"; import { getEngineNativeType, formatJsonQuery } from "metabase/lib/engine"; @@ -1438,6 +1438,35 @@ export const loadObjectDetailFKReferences = createThunkAction( }, ); +const ADD_FIELD = "metabase/qb/ADD_FIELD"; +export const addField = createThunkAction( + ADD_FIELD, + (field, run = true) => (dispatch, getState) => { + const { qb: { card } } = getState(); + const queryResult = getFirstQueryResult(getState()); + if ( + card.dataset_query.type === "query" && + queryResult && + queryResult.data + ) { + dispatch( + setDatasetQuery( + { + ...card.dataset_query, + query: { + ...card.dataset_query.query, + fields: getExistingFields(card, queryResult.data.cols).concat([ + field, + ]), + }, + }, + true, + ), + ); + } + }, +); + // DEPRECATED: use metabase/entities/questions export const ARCHIVE_QUESTION = "metabase/qb/ARCHIVE_QUESTION"; export const archiveQuestion = createThunkAction( diff --git a/frontend/src/metabase/query_builder/components/VisualizationSettings.jsx b/frontend/src/metabase/query_builder/components/VisualizationSettings.jsx index 4d195d4c28b20..b22ba0f2dcb90 100644 --- a/frontend/src/metabase/query_builder/components/VisualizationSettings.jsx +++ b/frontend/src/metabase/query_builder/components/VisualizationSettings.jsx @@ -17,7 +17,7 @@ export default class VisualizationSettings extends React.Component { } static propTypes = { - card: PropTypes.object.isRequired, + question: PropTypes.object.isRequired, result: PropTypes.object, setDisplayFn: PropTypes.func.isRequired, onUpdateVisualizationSettings: PropTypes.func.isRequired, @@ -31,9 +31,9 @@ export default class VisualizationSettings extends React.Component { }; renderChartTypePicker() { - let { result, card } = this.props; + let { result, question } = this.props; let { CardVisualization } = getVisualizationRaw([ - { card, data: result.data }, + { card: question.card(), data: result.data }, ]); let triggerElement = ( @@ -68,7 +68,7 @@ export default class VisualizationSettings extends React.Component { className={cx( "p2 flex align-center cursor-pointer bg-brand-hover text-white-hover", { - "ChartType--selected": vizType === card.display, + "ChartType--selected": vizType === question.display(), "ChartType--notSensible": !( result && result.data && @@ -111,7 +111,14 @@ export default class VisualizationSettings extends React.Component { ref="popover" > diff --git a/frontend/src/metabase/visualizations/components/ChartSettings.jsx b/frontend/src/metabase/visualizations/components/ChartSettings.jsx index 1e49a9766f860..0e4ddbea9e651 100644 --- a/frontend/src/metabase/visualizations/components/ChartSettings.jsx +++ b/frontend/src/metabase/visualizations/components/ChartSettings.jsx @@ -23,12 +23,23 @@ const Widget = ({ value, onChange, props, + // NOTE: special props to support adding additional fields + question, + addField, }) => { const W = widget; return (
{title &&

{title}

} - {W && } + {W && ( + + )}
); }; @@ -102,7 +113,7 @@ class ChartSettings extends Component { }; render() { - const { isDashboard } = this.props; + const { isDashboard, question, addField } = this.props; const { series } = this.state; const tabs = {}; @@ -152,7 +163,12 @@ class ChartSettings extends Component {
{widgets && widgets.map(widget => ( - + ))}
diff --git a/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx b/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx index 94e9290306fba..54f8238587c05 100644 --- a/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx +++ b/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx @@ -2,9 +2,14 @@ import React, { Component } from "react"; import CheckBox from "metabase/components/CheckBox.jsx"; import Icon from "metabase/components/Icon.jsx"; +import PopoverWithTrigger from "metabase/components/PopoverWithTrigger"; +import FieldList from "metabase/query_builder/components/FieldList"; import { SortableContainer, SortableElement } from "react-sortable-hoc"; +import StructuredQuery from "metabase-lib/lib/queries/StructuredQuery"; +import { fieldRefForColumn } from "metabase/lib/dataset"; + import cx from "classnames"; import _ from "underscore"; @@ -75,8 +80,39 @@ export default class ChartSettingOrderedFields extends Component { }; render() { - const { value, columnNames } = this.props; + const { value, question, addField, columns, columnNames } = this.props; const anyEnabled = this.isAnySelected(); + + let additionalFieldsButton; + if (columns && question && question.query() instanceof StructuredQuery) { + const fieldRefs = columns.map(column => fieldRefForColumn(column)); + + additionalFieldsButton = ( + + {({ onClose }) => ( + { + const mbql = dimension.mbql(); + return !_.find(fieldRefs, fieldRef => + _.isEqual(fieldRef, mbql), + ); + })} + onFieldChange={field => { + addField(field); + onClose(); + }} + enableTimeGrouping={false} + /> + )} + + ); + } + return (
@@ -104,6 +140,7 @@ export default class ChartSettingOrderedFields extends Component { distance={5} helperClass="z5" /> + {additionalFieldsButton}
); } diff --git a/frontend/src/metabase/visualizations/visualizations/Table.jsx b/frontend/src/metabase/visualizations/visualizations/Table.jsx index 0ec7fef9f0ab3..5416a8e44a4a7 100644 --- a/frontend/src/metabase/visualizations/visualizations/Table.jsx +++ b/frontend/src/metabase/visualizations/visualizations/Table.jsx @@ -201,6 +201,7 @@ export default class Table extends Component { enabled: col.visibility_type !== "details-only", })), getProps: ([{ data: { cols } }]) => ({ + columns: cols, columnNames: cols.reduce( (o, col) => ({ ...o, [col.name]: getFriendlyName(col) }), {}, From 9952501c92a8291ff760b009bcd692255ada0c76 Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Tue, 10 Jul 2018 10:44:00 -0700 Subject: [PATCH 049/116] address sameer's feedback --- .../automagic_dashboards/metric/GenericMetric.yaml | 12 +++++++----- .../automagic_dashboards/table/GenericTable.yaml | 2 +- src/metabase/automagic_dashboards/core.clj | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/resources/automagic_dashboards/metric/GenericMetric.yaml b/resources/automagic_dashboards/metric/GenericMetric.yaml index 1968ab203fd16..f76fa1582a60e 100644 --- a/resources/automagic_dashboards/metric/GenericMetric.yaml +++ b/resources/automagic_dashboards/metric/GenericMetric.yaml @@ -42,8 +42,10 @@ groups: title: How this metric is distributed across different categories - Numbers: title: How this metric is distributed across different numbers - - LargeCategories: - title: The top and bottom for the [[this]] + - LargeCategoriesTop: + title: Top 5 per category + - LargeCategoriesBottom: + title: Bottom 5 per category dashboard_filters: - Timestamp - State @@ -119,7 +121,7 @@ cards: map.region: us_states - ByNumber: group: Numbers - title: How [[this]] is distributed across [[GenericNumber]] + title: "[[this]] by [[GenericNumber]]" metrics: this dimensions: - GenericNumber: @@ -153,7 +155,7 @@ cards: order_by: this: descending - ByCategoryLargeTop: - group: LargeCategories + group: LargeCategoriesTop title: "[[this]] per [[GenericCategoryLarge]], top 5" metrics: this dimensions: GenericCategoryLarge @@ -162,7 +164,7 @@ cards: this: descending limit: 5 - ByCategoryLargeBottom: - group: LargeCategories + group: LargeCategoriesBottom title: "[[this]] per [[GenericCategoryLarge]], bottom 5" metrics: this dimensions: GenericCategoryLarge diff --git a/resources/automagic_dashboards/table/GenericTable.yaml b/resources/automagic_dashboards/table/GenericTable.yaml index 481e93447ad0d..d06fc4986ef3c 100644 --- a/resources/automagic_dashboards/table/GenericTable.yaml +++ b/resources/automagic_dashboards/table/GenericTable.yaml @@ -128,7 +128,7 @@ cards: group: Overview # General - NumberDistribution: - title: How [[this.short-name]] are distributed across [[GenericNumber]] + title: "[[this.short-name]] by [[GenericNumber]]" dimensions: - GenericNumber: aggregation: default diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index f3b5389095058..cfd405f66a610 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -66,7 +66,7 @@ :avg (tru "average") :min (tru "minumum") :max (tru "maximum") - :count (tru "count") + :count (tru "number") :distinct (tru "distinct count") :stddev (tru "standard deviation") :cum-count (tru "cumulative count") @@ -1047,7 +1047,7 @@ cell-query (assoc :url cell-url)) opts)) (decompose-question root query opts)) - cell-query (merge (let [title (tru "A closer look at {0}" (cell-title root cell-query))] + cell-query (merge (let [title (tru "A closer look at the {0}" (cell-title root cell-query))] {:transient_name title :name title}))))))) From 8ec6309587f64768f9aec0bf72b889cf85608ef6 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 10 Jul 2018 12:44:26 -0700 Subject: [PATCH 050/116] Implement new 'additional fields' design --- frontend/src/metabase/lib/dataset.js | 20 +- .../components/ChartSettings.jsx | 19 +- .../settings/ChartSettingOrderedFields.jsx | 221 ++++++++++-------- .../visualizations/visualizations/Table.jsx | 2 +- 4 files changed, 148 insertions(+), 114 deletions(-) diff --git a/frontend/src/metabase/lib/dataset.js b/frontend/src/metabase/lib/dataset.js index 5600ed6d3605a..b6cfe006f37c3 100644 --- a/frontend/src/metabase/lib/dataset.js +++ b/frontend/src/metabase/lib/dataset.js @@ -71,13 +71,18 @@ export function findColumnForColumnSetting( columns: Column[], columnSetting: ColumnSetting, ): ?Column { - return _.find( - columns, - column => - (columnSetting.fieldRef && - _.isEqual(columnSetting.fieldRef, fieldRefForColumn(column))) || - columnSetting.name === column.name, - ); + const { fieldRef } = columnSetting; + // first try to find by fieldRef + if (fieldRef != null) { + const column = _.find(columns, col => + _.isEqual(fieldRef, fieldRefForColumn(col)), + ); + if (column) { + return column; + } + } + // if that fails, find by column name + return _.findWhere(columns, { name: columnSetting.name }); } /** @@ -110,6 +115,7 @@ export function getExistingFields(card: Card, cols: Column[]): ConcreteField[] { if (query.fields && query.fields > 0) { return query.fields; } else if (Q.isBareRows(query)) { + // $FlowFixMe: return cols.map(col => fieldRefForColumn(col)).filter(id => id != null); } else { return []; diff --git a/frontend/src/metabase/visualizations/components/ChartSettings.jsx b/frontend/src/metabase/visualizations/components/ChartSettings.jsx index 0e4ddbea9e651..2311f840c63f5 100644 --- a/frontend/src/metabase/visualizations/components/ChartSettings.jsx +++ b/frontend/src/metabase/visualizations/components/ChartSettings.jsx @@ -55,17 +55,14 @@ class ChartSettings extends Component { }; } - getChartTypeName() { - let { CardVisualization } = getVisualizationTransformed(this.props.series); - switch (CardVisualization.identifier) { - case "table": - return "table"; - case "scalar": - return "number"; - case "funnel": - return "funnel"; - default: - return "chart"; + componentWillReceiveProps(nextProps) { + if (this.props.series !== nextProps.series) { + this.setState({ + series: this._getSeries( + nextProps.series, + nextProps.series[0].card.visualization_settings, + ), + }); } } diff --git a/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx b/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx index 54f8238587c05..81ada5a852a9f 100644 --- a/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx +++ b/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx @@ -1,4 +1,5 @@ import React, { Component } from "react"; +import { t } from "c-3po"; import CheckBox from "metabase/components/CheckBox.jsx"; import Icon from "metabase/components/Icon.jsx"; @@ -8,44 +9,35 @@ import FieldList from "metabase/query_builder/components/FieldList"; import { SortableContainer, SortableElement } from "react-sortable-hoc"; import StructuredQuery from "metabase-lib/lib/queries/StructuredQuery"; -import { fieldRefForColumn } from "metabase/lib/dataset"; +import { + fieldRefForColumn, + findColumnForColumnSetting, +} from "metabase/lib/dataset"; +import { getFriendlyName } from "metabase/visualizations/lib/utils"; import cx from "classnames"; import _ from "underscore"; -const SortableField = SortableElement( - ({ field, columnNames, onSetEnabled }) => ( -
- onSetEnabled(e.target.checked)} - /> - {columnNames[field.name]} - -
+const SortableColumn = SortableElement( + ({ columnSetting, getColumnName, onRemove }) => ( + onRemove(columnSetting)} + /> ), ); -const SortableFieldList = SortableContainer( - ({ fields, columnNames, onSetEnabled }) => { +const SortableColumnList = SortableContainer( + ({ columnSettings, getColumnName, onRemove }) => { return (
- {fields.map((field, index) => ( - ( + onSetEnabled(index, enabled)} + index={columnSetting.index} + columnSetting={columnSetting} + getColumnName={getColumnName} + onRemove={onRemove} /> ))}
@@ -53,19 +45,19 @@ const SortableFieldList = SortableContainer( }, ); -export default class ChartSettingOrderedFields extends Component { - handleSetEnabled = (index, checked) => { - const fields = [...this.props.value]; - fields[index] = { ...fields[index], enabled: checked }; - this.props.onChange(fields); +export default class ChartSettingOrderedColumns extends Component { + handleEnable = columnSetting => { + const columnSettings = [...this.props.value]; + const index = columnSetting.index; + columnSettings[index] = { ...columnSettings[index], enabled: true }; + this.props.onChange(columnSettings); }; - handleToggleAll = anyEnabled => { - const fields = this.props.value.map(field => ({ - ...field, - enabled: !anyEnabled, - })); - this.props.onChange([...fields]); + handleDisable = columnSetting => { + const columnSettings = [...this.props.value]; + const index = columnSetting.index; + columnSettings[index] = { ...columnSettings[index], enabled: false }; + this.props.onChange(columnSettings); }; handleSortEnd = ({ oldIndex, newIndex }) => { @@ -74,74 +66,113 @@ export default class ChartSettingOrderedFields extends Component { this.props.onChange(fields); }; - isAnySelected = () => { - const { value } = this.props; - return _.any(value, field => field.enabled); - }; + getColumnName = columnSetting => + getFriendlyName( + findColumnForColumnSetting(this.props.columns, columnSetting) || { + display_name: "[Unknown]", + }, + ); render() { const { value, question, addField, columns, columnNames } = this.props; - const anyEnabled = this.isAnySelected(); - let additionalFieldsButton; + let additionalFieldOptions = { count: 0 }; if (columns && question && question.query() instanceof StructuredQuery) { const fieldRefs = columns.map(column => fieldRefForColumn(column)); - - additionalFieldsButton = ( - - {({ onClose }) => ( - { - const mbql = dimension.mbql(); - return !_.find(fieldRefs, fieldRef => - _.isEqual(fieldRef, mbql), - ); - })} - onFieldChange={field => { - addField(field); - onClose(); - }} - enableTimeGrouping={false} - /> - )} - - ); + additionalFieldOptions = question.query().dimensionOptions(dimension => { + const mbql = dimension.mbql(); + return !_.find(fieldRefs, fieldRef => _.isEqual(fieldRef, mbql)); + }); } + const [enabledColumns, disabledColumns] = _.partition( + value.map((columnSetting, index) => ({ ...columnSetting, index })), + columnSetting => columnSetting.enabled, + ); + return (
-
-
- this.handleToggleAll(anyEnabled)} - invertChecked - /> - - {anyEnabled ? "Unselect all" : "Select all"} - +
{t`Click and drag to change their order`}
+ {enabledColumns.length > 0 ? ( + + ) : ( +
+ {t`Add fields from the list below`}
-
- - {additionalFieldsButton} + )} + {disabledColumns.length > 0 || additionalFieldOptions.count > 0 ? ( +

{`More fields`}

+ ) : null} + {disabledColumns.map(columnSetting => ( + this.handleEnable(columnSetting)} + /> + ))} + {additionalFieldOptions.count > 0 && ( +
+ {additionalFieldOptions.dimensions.map(dimension => ( + addField(dimension.mbql())} + /> + ))} + {additionalFieldOptions.fks.map(fk => ( +
+
+ {fk.field.target.table.display_name} +
+ {fk.dimensions.map(dimension => ( + addField(dimension.mbql())} + /> + ))} +
+ ))} +
+ )}
); } } + +const ColumnItem = ({ title, onAdd, onRemove }) => ( +
+
+
+ {title} + {onAdd && ( + { + e.stopPropagation(); + onAdd(); + }} + /> + )} + {onRemove && ( + { + e.stopPropagation(); + onRemove(); + }} + /> + )} +
+
+
+); diff --git a/frontend/src/metabase/visualizations/visualizations/Table.jsx b/frontend/src/metabase/visualizations/visualizations/Table.jsx index 5416a8e44a4a7..6021d19f708c9 100644 --- a/frontend/src/metabase/visualizations/visualizations/Table.jsx +++ b/frontend/src/metabase/visualizations/visualizations/Table.jsx @@ -186,7 +186,7 @@ export default class Table extends Component { }, "table.columns": { section: "Data", - title: t`Fields to include`, + title: t`Visible fields`, widget: ChartSettingOrderedFields, getHidden: (series, vizSettings) => vizSettings["table.pivot"], isValid: ([{ card, data }]) => From c2c4e2d77e378be27015f6107b602eff944a7b06 Mon Sep 17 00:00:00 2001 From: Cam Saul Date: Tue, 10 Jul 2018 12:22:05 -0700 Subject: [PATCH 051/116] MetaBot tweaks for multiple instances --- src/metabase/api/session.clj | 17 ++- src/metabase/db.clj | 95 +++++++------- src/metabase/events.clj | 1 + src/metabase/metabot.clj | 212 +++++++++++++++++++++++++++----- src/metabase/models/setting.clj | 98 +++++++++++---- src/metabase/util/date.clj | 85 ++++++++----- 6 files changed, 365 insertions(+), 143 deletions(-) diff --git a/src/metabase/api/session.clj b/src/metabase/api/session.clj index eeb578f5373db..22e80070ba807 100644 --- a/src/metabase/api/session.clj +++ b/src/metabase/api/session.clj @@ -33,7 +33,8 @@ (db/insert! Session :id <> :user_id (:id user)) - (events/publish-event! :user-login {:user_id (:id user), :session_id <>, :first_login (not (boolean (:last_login user)))}))) + (events/publish-event! :user-login + {:user_id (:id user), :session_id <>, :first_login (not (boolean (:last_login user)))}))) ;;; ## API Endpoints @@ -53,7 +54,8 @@ (try (when-let [user-info (ldap/find-user username)] (when-not (ldap/verify-password user-info password) - ;; Since LDAP knows about the user, fail here to prevent the local strategy to be tried with a possibly outdated password + ;; Since LDAP knows about the user, fail here to prevent the local strategy to be tried with a possibly + ;; outdated password (throw (ex-info password-fail-message {:status-code 400 :errors {:password password-fail-snippet}}))) @@ -114,7 +116,8 @@ (throttle/check (forgot-password-throttlers :ip-address) remote-address) (throttle/check (forgot-password-throttlers :email) email) ;; Don't leak whether the account doesn't exist, just pretend everything is ok - (when-let [{user-id :id, google-auth? :google_auth} (db/select-one ['User :id :google_auth] :email email, :is_active true)] + (when-let [{user-id :id, google-auth? :google_auth} (db/select-one ['User :id :google_auth] + :email email, :is_active true)] (let [reset-token (user/set-password-reset-token! user-id) password-reset-url (str (public-settings/site-url) "/auth/reset_password/" reset-token)] (email/send-password-reset-email! email google-auth? server-name password-reset-url) @@ -130,7 +133,8 @@ [^String token] (when-let [[_ user-id] (re-matches #"(^\d+)_.+$" token)] (let [user-id (Integer/parseInt user-id)] - (when-let [{:keys [reset_token reset_triggered], :as user} (db/select-one [User :id :last_login :reset_triggered :reset_token] + (when-let [{:keys [reset_token reset_triggered], :as user} (db/select-one [User :id :last_login :reset_triggered + :reset_token] :id user-id, :is_active true)] ;; Make sure the plaintext token matches up with the hashed one for this user (when (u/ignore-exceptions @@ -206,8 +210,9 @@ (when-not (autocreate-user-allowed-for-email? email) ;; Use some wacky status code (428 - Precondition Required) so we will know when to so the error screen specific ;; to this situation - (throw (ex-info (tru "You''ll need an administrator to create a Metabase account before you can use Google to log in.") - {:status-code 428})))) + (throw + (ex-info (tru "You''ll need an administrator to create a Metabase account before you can use Google to log in.") + {:status-code 428})))) (s/defn ^:private google-auth-create-new-user! [{:keys [email] :as new-user} :- user/NewUser] diff --git a/src/metabase/db.clj b/src/metabase/db.clj index 78110be4483e7..2e0c5e3cea538 100644 --- a/src/metabase/db.clj +++ b/src/metabase/db.clj @@ -1,5 +1,5 @@ (ns metabase.db - "Database definition and helper functions for interacting with the database." + "Application database definition, and setup logic, and helper functions for interacting with it." (:require [clojure [string :as s] [walk :as walk]] @@ -10,6 +10,7 @@ [metabase [config :as config] [util :as u]] + [puppetlabs.i18n.core :refer [trs]] [metabase.db.spec :as dbspec] [ring.util.codec :as codec] [toucan.db :as db]) @@ -45,8 +46,9 @@ [(System/getProperty "user.dir") "/" db-file-name options])))))) (defn- parse-connection-string - "Parse a DB connection URI like `postgres://cam@localhost.com:5432/cams_cool_db?ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory` - and return a broken-out map." + "Parse a DB connection URI like + `postgres://cam@localhost.com:5432/cams_cool_db?ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory` and + return a broken-out map." [uri] (when-let [[_ protocol user pass host port db query] (re-matches #"^([^:/@]+)://(?:([^:/@]+)(?::([^:@]+))?@)?([^:@]+)(?::(\d+))?/([^/?]+)(?:\?(.*))?$" uri)] (merge {:type (case (keyword protocol) @@ -73,8 +75,8 @@ (config/config-kw :mb-db-type))) (def db-connection-details - "Connection details that can be used when pretending the Metabase DB is itself a `Database` - (e.g., to use the Generic SQL driver functions on the Metabase DB itself)." + "Connection details that can be used when pretending the Metabase DB is itself a `Database` (e.g., to use the Generic + SQL driver functions on the Metabase DB itself)." (delay (or @connection-string-details (case (db-type) :h2 {:type :h2 ; TODO - we probably don't need to specifc `:type` here since we can just call (db-type) @@ -112,19 +114,19 @@ (def ^:private ^:const ^String changelog-file "liquibase.yaml") (defn- migrations-sql - "Return a string of SQL containing the DDL statements needed to perform unrun LIQUIBASE migrations." + "Return a string of SQL containing the DDL statements needed to perform unrun `liquibase` migrations." ^String [^Liquibase liquibase] (let [writer (StringWriter.)] (.update liquibase "" writer) (.toString writer))) (defn- migrations-lines - "Return a sequnce of DDL statements that should be used to perform migrations for LIQUIBASE. + "Return a sequnce of DDL statements that should be used to perform migrations for `liquibase`. - MySQL gets snippy if we try to run the entire DB migration as one single string; it seems to only like it if we run - one statement at a time; Liquibase puts each DDL statement on its own line automatically so just split by lines and - filter out blank / comment lines. Even though this is not neccesary for H2 or Postgres go ahead and do it anyway - because it keeps the code simple and doesn't make a significant performance difference." + MySQL gets snippy if we try to run the entire DB migration as one single string; it seems to only like it if we run + one statement at a time; Liquibase puts each DDL statement on its own line automatically so just split by lines and + filter out blank / comment lines. Even though this is not neccesary for H2 or Postgres go ahead and do it anyway + because it keeps the code simple and doesn't make a significant performance difference." [^Liquibase liquibase] (for [line (s/split-lines (migrations-sql liquibase)) :when (not (or (s/blank? line) @@ -132,57 +134,61 @@ line)) (defn- has-unrun-migrations? - "Does LIQUIBASE have migration change sets that haven't been run yet? + "Does `liquibase` have migration change sets that haven't been run yet? - It's a good idea to Check to make sure there's actually something to do before running `(migrate :up)` because - `migrations-sql` will always contain SQL to create and release migration locks, which is both slightly dangerous - and a waste of time when we won't be using them." + It's a good idea to Check to make sure there's actually something to do before running `(migrate :up)` because + `migrations-sql` will always contain SQL to create and release migration locks, which is both slightly dangerous and + a waste of time when we won't be using them." ^Boolean [^Liquibase liquibase] (boolean (seq (.listUnrunChangeSets liquibase nil)))) (defn- has-migration-lock? - "Is a migration lock in place for LIQUIBASE?" + "Is a migration lock in place for `liquibase`?" ^Boolean [^Liquibase liquibase] (boolean (seq (.listLocks liquibase)))) (defn- wait-for-migration-lock-to-be-cleared - "Check and make sure the database isn't locked. If it is, sleep for 2 seconds and then retry several times. - There's a chance the lock will end up clearing up so we can run migrations normally." + "Check and make sure the database isn't locked. If it is, sleep for 2 seconds and then retry several times. There's a + chance the lock will end up clearing up so we can run migrations normally." [^Liquibase liquibase] (u/auto-retry 5 (when (has-migration-lock? liquibase) (Thread/sleep 2000) - (throw (Exception. (str "Database has migration lock; cannot run migrations. You can force-release these locks " - "by running `java -jar metabase.jar migrate release-locks`.")))))) + (throw + (Exception. + (str + (trs "Database has migration lock; cannot run migrations.") + " " + (trs "You can force-release these locks by running `java -jar metabase.jar migrate release-locks`."))))))) (defn- migrate-up-if-needed! - "Run any unrun LIQUIBASE migrations, if needed. + "Run any unrun `liquibase` migrations, if needed. - This creates SQL for the migrations to be performed, then executes each DDL statement. - Running `.update` directly doesn't seem to work as we'd expect; it ends up commiting the changes made and they - can't be rolled back at the end of the transaction block. Converting the migration to SQL string and running that - via `jdbc/execute!` seems to do the trick." + This creates SQL for the migrations to be performed, then executes each DDL statement. Running `.update` directly + doesn't seem to work as we'd expect; it ends up commiting the changes made and they can't be rolled back at the end + of the transaction block. Converting the migration to SQL string and running that via `jdbc/execute!` seems to do + the trick." [conn, ^Liquibase liquibase] - (log/info "Checking if Database has unrun migrations...") + (log/info (trs "Checking if Database has unrun migrations...")) (when (has-unrun-migrations? liquibase) - (log/info "Database has unrun migrations. Waiting for migration lock to be cleared...") + (log/info (trs "Database has unrun migrations. Waiting for migration lock to be cleared...")) (wait-for-migration-lock-to-be-cleared liquibase) - (log/info "Migration lock is cleared. Running migrations...") + (log/info (trs "Migration lock is cleared. Running migrations...")) (doseq [line (migrations-lines liquibase)] (jdbc/execute! conn [line])))) (defn- force-migrate-up-if-needed! "Force migrating up. This does two things differently from `migrate-up-if-needed!`: - 1. This doesn't check to make sure the DB locks are cleared - 2. Any DDL statements that fail are ignored + 1. This doesn't check to make sure the DB locks are cleared + 2. Any DDL statements that fail are ignored - It can be used to fix situations where the database got into a weird state, as was common before the fixes made in - #3295. + It can be used to fix situations where the database got into a weird state, as was common before the fixes made in + #3295. - Each DDL statement is ran inside a nested transaction; that way if the nested transaction fails we can roll it back - without rolling back the entirety of changes that were made. (If a single statement in a transaction fails you - can't do anything futher until you clear the error state by doing something like calling `.rollback`.)" + Each DDL statement is ran inside a nested transaction; that way if the nested transaction fails we can roll it back + without rolling back the entirety of changes that were made. (If a single statement in a transaction fails you can't + do anything futher until you clear the error state by doing something like calling `.rollback`.)" [conn, ^Liquibase liquibase] (.clearCheckSums liquibase) (when (has-unrun-migrations? liquibase) @@ -197,7 +203,7 @@ (def ^{:arglists '([])} ^DatabaseFactory database-factory "Return an instance of the Liquibase `DatabaseFactory`. This is done on a background thread at launch because - otherwise it adds 2 seconds to startup time." + otherwise it adds 2 seconds to startup time." (partial deref (future (DatabaseFactory/getInstance)))) (defn- conn->liquibase @@ -222,7 +228,8 @@ "DATABASECHANGELOG" "databasechangelog") fresh-install? (jdbc/with-db-metadata [meta (jdbc-details)] ;; don't migrate on fresh install - (empty? (jdbc/metadata-query (.getTables meta nil nil liquibases-table-name (into-array String ["TABLE"]))))) + (empty? (jdbc/metadata-query + (.getTables meta nil nil liquibases-table-name (into-array String ["TABLE"]))))) query (format "UPDATE %s SET FILENAME = ?" liquibases-table-name)] (when-not fresh-install? (jdbc/execute! conn [query "migrations/000_migrations.yaml"])))) @@ -252,11 +259,11 @@ ;; Disable auto-commit. This should already be off but set it just to be safe (.setAutoCommit (jdbc/get-connection conn) false) ;; Set up liquibase and let it do its thing - (log/info "Setting up Liquibase...") + (log/info (trs "Setting up Liquibase...")) (try (let [liquibase (conn->liquibase conn)] (consolidate-liquibase-changesets conn) - (log/info "Liquibase is ready.") + (log/info (trs "Liquibase is ready.")) (case direction :up (migrate-up-if-needed! conn liquibase) :force (force-migrate-up-if-needed! conn liquibase) @@ -279,7 +286,7 @@ ;;; +----------------------------------------------------------------------------------------------------------------+ (defn connection-pool - "Create a C3P0 connection pool for the given database SPEC." + "Create a C3P0 connection pool for the given database `spec`." [{:keys [subprotocol subname classname minimum-pool-size idle-connection-test-period excess-timeout] :or {minimum-pool-size 3 idle-connection-test-period 0 @@ -348,12 +355,12 @@ (verify-db-connection (:type db-details) db-details)) ([engine details] {:pre [(keyword? engine) (map? details)]} - (log/info (u/format-color 'cyan "Verifying %s Database Connection ..." (name engine))) + (log/info (u/format-color 'cyan (trs "Verifying {0} Database Connection ..." (name engine)))) (assert (binding [*allow-potentailly-unsafe-connections* true] (require 'metabase.driver) ((resolve 'metabase.driver/can-connect-with-details?) engine details)) (format "Unable to connect to Metabase %s DB." (name engine))) - (log/info "Verify Database Connection ... " (u/emoji "✅")))) + (log/info (trs "Verify Database Connection ... ") (u/emoji "✅")))) (def ^:dynamic ^Boolean *disable-data-migrations* @@ -379,11 +386,11 @@ (defn- run-schema-migrations! "Run through our DB migration process and make sure DB is fully prepared" [auto-migrate? db-details] - (log/info "Running Database Migrations...") + (log/info (trs "Running Database Migrations...")) (if auto-migrate? (migrate! db-details :up) (print-migrations-and-quit! db-details)) - (log/info "Database Migrations Current ... " (u/emoji "✅"))) + (log/info (trs "Database Migrations Current ... ") (u/emoji "✅"))) (defn- run-data-migrations! "Do any custom code-based migrations now that the db structure is up to date." diff --git a/src/metabase/events.clj b/src/metabase/events.clj index 28e245f5b2f51..11613ae0531e4 100644 --- a/src/metabase/events.clj +++ b/src/metabase/events.clj @@ -55,6 +55,7 @@ (defn publish-event! "Publish an item into the events stream. Returns the published item to allow for chaining." + {:style/indent 1} [topic event-item] {:pre [(keyword topic)]} (async/go (async/>! events-channel {:topic (keyword topic), :item event-item})) diff --git a/src/metabase/metabot.clj b/src/metabase/metabot.clj index 6b034ba180fad..216752084dc65 100644 --- a/src/metabase/metabot.clj +++ b/src/metabase/metabot.clj @@ -7,6 +7,7 @@ [string :as str]] [clojure.java.io :as io] [clojure.tools.logging :as log] + [honeysql.core :as hsql] [manifold [deferred :as d] [stream :as s]] @@ -21,8 +22,10 @@ [permissions :refer [Permissions]] [permissions-group :as perms-group] [setting :as setting :refer [defsetting]]] - [metabase.util.urls :as urls] - [puppetlabs.i18n.core :refer [tru trs]] + [metabase.util + [date :as du] + [urls :as urls]] + [puppetlabs.i18n.core :refer [trs tru]] [throttle.core :as throttle] [toucan.db :as db])) @@ -32,7 +35,106 @@ :default false) -;;; ------------------------------------------------------------ Perms Checking ------------------------------------------------------------ +;;; ------------------------------------- Deciding which instance is the MetaBot ------------------------------------- + +;; Close your eyes, and imagine a scenario: someone is running multiple Metabase instances in a horizontal cluster. +;; Good for them, but how do we make sure one, and only one, of those instances, replies to incoming MetaBot commands? +;; It would certainly be too much if someone ran, say, 4 instances, and typing `metabot kanye` into Slack gave them 4 +;; Kanye West quotes, wouldn't it? +;; +;; Luckily, we have an "elegant" solution: we'll use the Settings framework to keep track of which instance is +;; currently serving as the MetaBot. We'll have that instance periodically check in; if it doesn't check in for some +;; timeout interval, we'll consider the job of MetaBot up for grabs. Each instance will periodically check if the +;; MetaBot job is open, and, if so, whoever discovers it first will take it. + + +;; How do we uniquiely identify each instance? +;; +;; `local-process-uuid` is randomly-generated upon launch and used to identify this specific Metabase instance during +;; this specifc run. Restarting the server will change this UUID, and each server in a hortizontal cluster will have +;; its own ID, making this different from the `site-uuid` Setting. The local process UUID is used to differentiate +;; different horizontally clustered MB instances so we can determine which of them will handle MetaBot duties. +;; +;; TODO - if we ever want to use this elsewhere, we need to move it to `metabase.config` or somewhere else central +;; like that. +(defonce ^:private local-process-uuid + (str (java.util.UUID/randomUUID))) + +(defsetting ^:private metabot-instance-uuid + "UUID of the active MetaBot instance (the Metabase process currently handling MetaBot duties.)" + ;; This should be cached because we'll be checking it fairly often, basically every 2 seconds as part of the + ;; websocket monitor thread to see whether we're MetaBot (the thread won't open the WebSocket unless that instance + ;; is handling MetaBot duties) + :internal? true) + +(defsetting ^:private metabot-instance-last-checkin + "Timestamp of the last time the active MetaBot instance checked in." + :internal? true + ;; caching is disabled for this, since it is intended to be updated frequently (once a minute or so) If we use the + ;; cache, it will trigger cache invalidation for all the other instances (wasteful), and possibly at any rate be + ;; incorrect (for example, if another instance checked in a minute ago, our local cache might not get updated right + ;; away, causing us to falsely assume the MetaBot role is up for grabs.) + :cache? false + :type :timestamp) + +(defn- current-timestamp-from-db + "Fetch the current timestamp from the DB. Why do this from the DB? It's not safe to assume multiple instances have + clocks exactly in sync; but since each instance is using the same application DB, we can use it as a cannonical + source of truth." + ^java.sql.Timestamp [] + (-> (db/query {:select [[(hsql/raw "current_timestamp") :current_timestamp]]}) + first + :current_timestamp)) + +(defn- update-last-checkin! + "Update the last checkin timestamp recorded in the DB." + [] + (metabot-instance-last-checkin (current-timestamp-from-db))) + +(defn- seconds-since-last-checkin + "Return the number of seconds since the active MetaBot instance last checked in (updated the + `metabot-instance-last-checkin` Setting). If a MetaBot instance has *never* checked in, this returns `nil`. (Since + `last-checkin` is one of the few Settings that isn't cached, this always requires a DB call.)" + [] + (when-let [last-checkin (metabot-instance-last-checkin)] + (u/prog1 (-> (- (.getTime (current-timestamp-from-db)) + (.getTime last-checkin)) + (/ 1000)) + (log/debug (u/format-color 'magenta (trs "Last MetaBot checkin was {0} ago." (du/format-seconds <>))))))) + +(def ^:private ^Integer recent-checkin-timeout-interval-seconds + "Number of seconds since the last MetaBot checkin that we will consider the MetaBot job to be 'up for grabs', + currently 2 minutes. (i.e. if the current MetaBot job holder doesn't check in for more than 2 minutes, it's up for + grabs.)" + (int (* 60 2))) + +(defn- last-checkin-was-not-recent? + "`true` if the last checkin of the active MetaBot instance was more than 2 minutes ago, or if there has never been a + checkin. (This requires DB calls, so it should not be called too often -- once a minute [at the time of this + writing] should be sufficient.)" + [] + (if-let [seconds-since-last-checkin (seconds-since-last-checkin)] + (> seconds-since-last-checkin + recent-checkin-timeout-interval-seconds) + true)) + +(defn- am-i-the-metabot? + "Does this instance currently have the MetaBot job? (Does not require any DB calls, so may safely be called + often (i.e. in the websocket monitor thread loop.)" + [] + (= (metabot-instance-uuid) + local-process-uuid)) + +(defn- become-metabot! + "Direct this instance to assume the duties of acting as MetaBot, and update the Settings we use to track assignment + accordingly." + [] + (log/info (u/format-color 'green (trs "This instance will now handle MetaBot duties."))) + (metabot-instance-uuid local-process-uuid) + (update-last-checkin!)) + + +;;; ------------------------------------------------- Perms Checking ------------------------------------------------- (defn- metabot-permissions "Return the set of permissions granted to the MetaBot." @@ -50,7 +152,7 @@ `(do-with-metabot-permissions (fn [] ~@body))) -;;; # ------------------------------------------------------------ Metabot Command Handlers ------------------------------------------------------------ +;;; -------------------------------------------- Metabot Command Handlers -------------------------------------------- (def ^:private ^:dynamic *channel-id* nil) @@ -110,15 +212,18 @@ (defn- card-with-name [card-name] (first (u/prog1 (db/select [Card :id :name], :%lower.name [:like (str \% (str/lower-case card-name) \%)]) (when (> (count <>) 1) - (throw (Exception. (str (tru "Could you be a little more specific? I found these cards with names that matched:\n{0}" - (format-cards <>))))))))) + (throw (Exception. + (str (tru "Could you be a little more specific? I found these cards with names that matched:\n{0}" + (format-cards <>))))))))) (defn- id-or-name->card [card-id-or-name] (cond (integer? card-id-or-name) (db/select-one [Card :id :name], :id card-id-or-name) (or (string? card-id-or-name) (symbol? card-id-or-name)) (card-with-name card-id-or-name) - :else (throw (Exception. (str (tru "I don''t know what Card `{0}` is. Give me a Card ID or name." card-id-or-name)))))) + :else (throw (Exception. + (str (tru "I don''t know what Card `{0}` is. Give me a Card ID or name." + card-id-or-name)))))) (defn ^:metabot show @@ -130,13 +235,16 @@ (do (with-metabot-permissions (read-check Card card-id)) - (do-async (let [attachments (pulse/create-and-upload-slack-attachments! (pulse/create-slack-attachment-data [(pulse/execute-card card-id, :context :metabot)]))] + (do-async (let [attachments (pulse/create-and-upload-slack-attachments! + (pulse/create-slack-attachment-data + [(pulse/execute-card card-id, :context :metabot)]))] (slack/post-chat-message! *channel-id* nil attachments))) (tru "Ok, just a second...")) (throw (Exception. (str (tru "Not Found")))))) - ;; If the card name comes without spaces, e.g. (show 'my 'wacky 'card) turn it into a string an recur: (show "my wacky card") + ;; If the card name comes without spaces, e.g. (show 'my 'wacky 'card) turn it into a string an recur: (show "my + ;; wacky card") ([word & more] (show (str/join " " (cons word more))))) @@ -171,7 +279,7 @@ (str ":kanye:\n> " (rand-nth @kanye-quotes))) -;;; # ------------------------------------------------------------ Metabot Command Dispatch ------------------------------------------------------------ +;;; -------------------------------------------- Metabot Command Dispatch -------------------------------------------- (def ^:private apply-metabot-fn (dispatch-fn "understand" :metabot)) @@ -189,7 +297,7 @@ (apply apply-metabot-fn tokens))))) -;;; # ------------------------------------------------------------ Metabot Input Handling ------------------------------------------------------------ +;;; --------------------------------------------- Metabot Input Handling --------------------------------------------- (defn- message->command-str "Get the command portion of a message *event* directed at Metabot. @@ -241,7 +349,7 @@ nil))))) -;;; # ------------------------------------------------------------ Websocket Connection Stuff ------------------------------------------------------------ +;;; ------------------------------------------- Websocket Connection Stuff ------------------------------------------- (defn- connect-websocket! [] (when-let [websocket-url (slack/websocket-url)] @@ -264,9 +372,10 @@ ;; and if it is no longer equal to theirs they should die (defonce ^:private websocket-monitor-thread-id (atom nil)) -;; we'll use a THROTTLER to implement exponential backoff for recconenction attempts, since THROTTLERS are designed with for this sort of thing -;; e.g. after the first failed connection we'll wait 2 seconds, then each that amount increases by the `:delay-exponent` of 1.3 -;; so our reconnection schedule will look something like: +;; we'll use a THROTTLER to implement exponential backoff for recconenction attempts, since THROTTLERS are designed +;; with for this sort of thing e.g. after the first failed connection we'll wait 2 seconds, then each that amount +;; increases by the `:delay-exponent` of 1.3. So our reconnection schedule will look something like: +;; ;; number of consecutive failed attempts | seconds before next try (rounded up to nearest multiple of 2 seconds) ;; --------------------------------------+---------------------------------------------------------------------- ;; 0 | 2 @@ -276,47 +385,82 @@ ;; 4 | 8 ;; 5 | 14 ;; 6 | 30 -;; we'll throttle this based on values of the `slack-token` setting; that way if someone changes its value they won't have to wait -;; whatever the exponential delay is before the connection is retried +;; +;; we'll throttle this based on values of the `slack-token` setting; that way if someone changes its value they won't +;; have to wait whatever the exponential delay is before the connection is retried (def ^:private reconnection-attempt-throttler (throttle/make-throttler nil :attempts-threshold 1, :initial-delay-ms 2000, :delay-exponent 1.3)) (defn- should-attempt-to-reconnect? ^Boolean [] - (boolean (u/ignore-exceptions - (throttle/check reconnection-attempt-throttler (slack/slack-token)) - true))) + (boolean + (u/ignore-exceptions + (throttle/check reconnection-attempt-throttler (slack/slack-token)) + true))) + +(defn- reopen-websocket-connection-if-needed! + "Check to see if websocket connection is [still] open, [re-]open it if not." + [] + ;; Only open the Websocket connection if this instance is the MetaBot + (when (am-i-the-metabot?) + (when (= (.getId (Thread/currentThread)) @websocket-monitor-thread-id) + (try + (when (or (not @websocket) + (s/closed? @websocket)) + (log/debug (trs "MetaBot WebSocket is closed. Reconnecting now.")) + (connect-websocket!)) + (catch Throwable e + (log/error (trs "Error connecting websocket:") (.getMessage e))))))) (defn- start-websocket-monitor! [] (future (reset! websocket-monitor-thread-id (.getId (Thread/currentThread))) - ;; Every 2 seconds check to see if websocket connection is [still] open, [re-]open it if not (loop [] + ;; Every 2 seconds... (while (not (should-attempt-to-reconnect?)) (Thread/sleep 2000)) - (when (= (.getId (Thread/currentThread)) @websocket-monitor-thread-id) - (try - (when (or (not @websocket) - (s/closed? @websocket)) - (log/debug (trs "MetaBot WebSocket is closed. Reconnecting now.")) - (connect-websocket!)) - (catch Throwable e - (log/error (trs "Error connecting websocket:") (.getMessage e)))) - (recur))))) + (reopen-websocket-connection-if-needed!) + (recur)))) + + +(defn- check-and-update-instance-status! + "Check whether the current instance is serving as the MetaBot; if so, update the last checkin timestamp; if not, check + whether we should become the MetaBot (and do so if we should)." + [] + (cond + ;; if we're already the MetaBot instance, update the last checkin timestamp + (am-i-the-metabot?) + (do + (log/debug (trs "This instance is performing MetaBot duties.")) + (update-last-checkin!)) + ;; otherwise if the last checkin was too long ago, it's time for us to assume the mantle of MetaBot + (last-checkin-was-not-recent?) + (become-metabot!) + ;; otherwise someone else is the MetaBot and we're done here! woo + :else + (log/debug (u/format-color 'blue (trs "Another instance is already handling MetaBot duties."))))) + +(defn- start-instance-monitor! [] + (future + (loop [] + (check-and-update-instance-status!) + (Thread/sleep (* 60 1000)) + (recur)))) (defn start-metabot! "Start the MetaBot! :robot_face: - This will spin up a background thread that opens and maintains a Slack WebSocket connection." + This will spin up a background thread that opens and maintains a Slack WebSocket connection." [] (when (and (slack/slack-token) (metabot-enabled)) - (log/info "Starting MetaBot WebSocket monitor thread...") - (start-websocket-monitor!))) + (log/info (trs "Starting MetaBot threads...")) + (start-websocket-monitor!) + (start-instance-monitor!))) (defn stop-metabot! "Stop the MetaBot! :robot_face: - This will stop the background thread that responsible for the Slack WebSocket connection." + This will stop the background thread that responsible for the Slack WebSocket connection." [] (log/info (trs "Stopping MetaBot... 🤖")) (reset! websocket-monitor-thread-id nil) diff --git a/src/metabase/models/setting.clj b/src/metabase/models/setting.clj index 29a595b9e26a6..2c28b92fc1a77 100644 --- a/src/metabase/models/setting.clj +++ b/src/metabase/models/setting.clj @@ -42,7 +42,9 @@ [db :as mdb] [events :as events] [util :as u]] - [metabase.util.honeysql-extensions :as hx] + [metabase.util + [date :as du] + [honeysql-extensions :as hx]] [puppetlabs.i18n.core :refer [trs tru]] [schema.core :as s] [toucan @@ -60,7 +62,16 @@ (def ^:private Type - (s/enum :string :boolean :json :integer :double)) + (s/enum :string :boolean :json :integer :double :timestamp)) + +(def ^:private default-tag-for-type + "Type tag that will be included in the Setting's metadata, so that the getter function will not cause reflection + warnings." + {:string String + :boolean Boolean + :integer Long + :double Double + :timestamp java.sql.Timestamp}) (def ^:private SettingDefinition {:name s/Keyword @@ -69,7 +80,9 @@ :type Type ; all values are stored in DB as Strings, :getter clojure.lang.IFn ; different getters/setters take care of parsing/unparsing :setter clojure.lang.IFn - :internal? s/Bool}) ; should the API never return this setting? (default: false) + :tag (s/maybe Class) ; type annotation, e.g. ^String, to be applied. Defaults to tag based on :type + :internal? s/Bool ; should the API never return this setting? (default: false) + :cache? s/Bool}) ; should the getter always fetch this value "fresh" from the DB? (default: false) (defonce ^:private registered-settings @@ -157,10 +170,13 @@ (when-let [last-known-update (core/get @cache settings-last-updated-key)] ;; compare it to the value in the DB. This is done be seeing whether a row exists ;; WHERE value > - (db/select-one Setting - {:where [:and - [:= :key settings-last-updated-key] - [:> :value last-known-update]]}))))) + (u/prog1 (db/select-one Setting + {:where [:and + [:= :key settings-last-updated-key] + [:> :value last-known-update]]}) + (when <> + (log/info (u/format-color 'red + (trs "Settings have been changed on another instance, and will be reloaded here."))))))))) (def ^:private cache-update-check-interval-ms "How often we should check whether the Settings cache is out of date (which requires a DB call)?" @@ -227,11 +243,16 @@ (when (seq v) v))) +(def ^:private ^:dynamic *disable-cache* false) + (defn- db-value "Get the value, if any, of `setting-or-name` from the DB (using / restoring the cache as needed)." ^String [setting-or-name] - (restore-cache-if-needed!) - (clojure.core/get @cache (setting-name setting-or-name))) + (if *disable-cache* + (db/select-one-field :value Setting :key (setting-name setting-or-name)) + (do + (restore-cache-if-needed!) + (clojure.core/get @cache (setting-name setting-or-name))))) (defn get-string @@ -285,18 +306,26 @@ [setting-or-name] (json/parse-string (get-string setting-or-name) keyword)) +(defn get-timestamp + "Get the string value of `setting-or-name` and parse it as an ISO-8601-formatted string, returning a Timestamp." + [setting-or-name] + (du/->Timestamp (get-string setting-or-name) :no-timezone)) + (def ^:private default-getter-for-type - {:string get-string - :boolean get-boolean - :integer get-integer - :json get-json - :double get-double}) + {:string get-string + :boolean get-boolean + :integer get-integer + :json get-json + :timestamp get-timestamp + :double get-double}) (defn get "Fetch the value of `setting-or-name`. What this means depends on the Setting's `:getter`; by default, this looks for first for a corresponding env var, then checks the cache, then returns the default value of the Setting, if any." [setting-or-name] - ((:getter (resolve-setting setting-or-name)))) + (let [{:keys [cache? getter]} (resolve-setting setting-or-name)] + (binding [*disable-cache* (not cache?)] + (getter)))) ;;; +----------------------------------------------------------------------------------------------------------------+ @@ -353,7 +382,10 @@ (swap! cache assoc setting-name new-value) (swap! cache dissoc setting-name)) ;; Record the fact that a Setting has been updated so eventaully other instances (if applicable) find out about it - (update-settings-last-updated!) + ;; (For Settings that don't use the Cache, don't update the `last-updated` value, because it will cause other + ;; instances to do needless reloading of the cache from the DB) + (when-not *disable-cache* + (update-settings-last-updated!)) ;; Now return the `new-value`. new-value)) @@ -389,15 +421,20 @@ (defn set-json! "Serialize `new-value` for `setting-or-name` as a JSON string and save it." [setting-or-name new-value] - (set-string! setting-or-name (when new-value - (json/generate-string new-value)))) + (set-string! setting-or-name (some-> new-value json/generate-string))) + +(defn set-timestamp! + "Serialize `new-value` for `setting-or-name` as a ISO 8601-encoded timestamp strign and save it." + [setting-or-name new-value] + (set-string! setting-or-name (some-> new-value du/date->iso-8601))) (def ^:private default-setter-for-type - {:string set-string! - :boolean set-boolean! - :integer set-integer! - :json set-json! - :double set-double!}) + {:string set-string! + :boolean set-boolean! + :integer set-integer! + :json set-json! + :timestamp set-timestamp! + :double set-double!}) (defn set! "Set the value of `setting-or-name`. What this means depends on the Setting's `:setter`; by default, this just updates @@ -409,7 +446,9 @@ (mandrill-api-key \"xyz123\")" [setting-or-name new-value] - ((:setter (resolve-setting setting-or-name)) new-value)) + (let [{:keys [setter cache?]} (resolve-setting setting-or-name)] + (binding [*disable-cache* (not cache?)] + (setter new-value)))) ;;; +----------------------------------------------------------------------------------------------------------------+ @@ -427,7 +466,9 @@ :default default :getter (partial (default-getter-for-type setting-type) setting-name) :setter (partial (default-setter-for-type setting-type) setting-name) - :internal? false} + :tag (default-tag-for-type setting-type) + :internal? false + :cache? true} (dissoc setting :name :type :default))) (s/validate SettingDefinition <>) (swap! registered-settings assoc setting-name <>))) @@ -440,10 +481,11 @@ (defn metadata-for-setting-fn "Create metadata for the function automatically generated by `defsetting`." - [{:keys [default description], setting-type :type, :as setting}] + [{:keys [default description tag], setting-type :type, :as setting}] {:arglists '([] [new-value]) ;; indentation below is intentional to make it clearer what shape the generated documentation is going to take. ;; Turn on auto-complete-mode in Emacs and see for yourself! + :tag tag :doc (str/join "\n" [ description "" (format "`%s` is a %s Setting. You can get its value by calling:" (setting-name setting) (name setting-type)) @@ -501,7 +543,9 @@ * `:setter` - A custom setter fn, which takes a single argument. Overrides the default implementation. (This can in turn call functions in this namespace like `set-string!` or `set-boolean!` to invoke the default setter behavior. Keep in mind that the custom setter may be passed `nil`, which should - clear the values of the Setting.)" + clear the values of the Setting.) + * `:cache?` - Should this Setting be cached? (default `true`)? Be careful when disabling this, because it could + have a very negative performance impact." {:style/indent 1} [setting-symb description & {:as options}] {:pre [(symbol? setting-symb)]} diff --git a/src/metabase/util/date.clj b/src/metabase/util/date.clj index 2ad043a84d29e..a932f866e96c3 100644 --- a/src/metabase/util/date.clj +++ b/src/metabase/util/date.clj @@ -24,7 +24,7 @@ :tag TimeZone} *data-timezone*) -(defprotocol ITimeZoneCoercible +(defprotocol ^:private ITimeZoneCoercible "Coerce object to `java.util.TimeZone`" (coerce-to-timezone ^TimeZone [this] "Coerce `this` to `java.util.TimeZone`")) @@ -85,7 +85,7 @@ [db & body] `(call-with-effective-timezone ~db (fn [] ~@body))) -(defprotocol ITimestampCoercible +(defprotocol ^:private ITimestampCoercible "Coerce object to a `java.sql.Timestamp`." (coerce-to-timestamp ^java.sql.Timestamp [this] [this timezone-coercible] "Coerce this object to a `java.sql.Timestamp`. Strings are parsed as ISO-8601.")) @@ -110,7 +110,12 @@ (defn ^Timestamp ->Timestamp "Converts `coercible-to-ts` to a `java.util.Timestamp`. Requires a `coercible-to-tz` if converting a string. Leans - on clj-time to ensure correct conversions between the various types" + on clj-time to ensure correct conversions between the various types + + NOTE: This function requires you to pass in a timezone or bind `*report-timezone*`, probably to make sure you're not + doing something dumb by forgetting it.For cases where you'd just like to parse an ISO-8601-encoded String in peace + without specifying a timezone, pass in `:no-timezone` as the second param to explicitly have things parsed without + one." ([coercible-to-ts] {:pre [(or (not (string? coercible-to-ts)) (and (string? coercible-to-ts) (bound? #'*report-timezone*)))]} @@ -119,10 +124,11 @@ {:pre [(or (not (string? coercible-to-ts)) (and (string? coercible-to-ts) timezone))]} (if (string? coercible-to-ts) - (coerce-to-timestamp (str->date-time coercible-to-ts (coerce-to-timezone timezone))) + (coerce-to-timestamp (str->date-time coercible-to-ts (when-not (= timezone :no-timezone) + (coerce-to-timezone timezone)))) (coerce-to-timestamp coercible-to-ts)))) -(defprotocol IDateTimeFormatterCoercible +(defprotocol ^:private IDateTimeFormatterCoercible "Protocol for converting objects to `DateTimeFormatters`." (->DateTimeFormatter ^org.joda.time.format.DateTimeFormatter [this] "Coerce object to a `DateTimeFormatter`.")) @@ -139,15 +145,15 @@ (defn parse-date - "Parse a datetime string S with a custom DATE-FORMAT, which can be a format string, clj-time formatter keyword, or + "Parse a datetime string `s` with a custom `date-format`, which can be a format string, clj-time formatter keyword, or anything else that can be coerced to a `DateTimeFormatter`. - (parse-date \"yyyyMMdd\" \"20160201\") -> #inst \"2016-02-01\" - (parse-date :date-time \"2016-02-01T00:00:00.000Z\") -> #inst \"2016-02-01\"" + (parse-date \"yyyyMMdd\" \"20160201\") -> #inst \"2016-02-01\" + (parse-date :date-time \"2016-02-01T00:00:00.000Z\") -> #inst \"2016-02-01\"" ^java.sql.Timestamp [date-format, ^String s] (->Timestamp (time/parse (->DateTimeFormatter date-format) s))) -(defprotocol ISO8601 +(defprotocol ^:private ISO8601 "Protocol for converting objects to ISO8601 formatted strings." (->iso-8601-datetime ^String [this timezone-id] "Coerce object to an ISO8601 date-time string such as \"2015-11-18T23:55:03.841Z\" with a given TIMEZONE.")) @@ -174,12 +180,12 @@ (time/formatters :time))))) (defn format-time - "Returns a string representation of the time found in `T`" + "Returns a string representation of the time found in `t`" [t time-zone-id] (time/unparse (time-formatter time-zone-id) (coerce/to-date-time t))) (defn is-time? - "Returns true if `V` is a Time object" + "Returns true if `v` is a Time object" [v] (and v (instance? Time v))) @@ -198,13 +204,13 @@ (->Timestamp (System/currentTimeMillis))) (defn format-date - "Format DATE using a given DATE-FORMAT. NOTE: This will create a date string in the JVM's timezone, not the report + "Format `date` using a given `date-format`. NOTE: This will create a date string in the JVM's timezone, not the report timezone. - DATE is anything that can coerced to a `Timestamp` via `->Timestamp`, such as a `Date`, `Timestamp`, - `Long` (ms since the epoch), or an ISO-8601 `String`. DATE defaults to the current moment in time. + `date` is anything that can coerced to a `Timestamp` via `->Timestamp`, such as a `Date`, `Timestamp`, + `Long` (ms since the epoch), or an ISO-8601 `String`. `date` defaults to the current moment in time. - DATE-FORMAT is anything that can be passed to `->DateTimeFormatter`, such as `String` + `date-format` is anything that can be passed to `->DateTimeFormatter`, such as `String` (using [the usual date format args](http://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html)), `Keyword`, or `DateTimeFormatter`. @@ -218,7 +224,7 @@ (time/unparse (->DateTimeFormatter date-format) (coerce/from-sql-time (->Timestamp date))))) (def ^{:arglists '([] [date])} date->iso-8601 - "Format DATE a an ISO-8601 string." + "Format `date` a an ISO-8601 string." (partial format-date :date-time)) (defn date-string? @@ -231,14 +237,14 @@ (->Timestamp s utc))))) (defn ->Date - "Coerece DATE to a `java.util.Date`." + "Coerece `date` to a `java.util.Date`." (^java.util.Date [] (java.util.Date.)) (^java.util.Date [date] (java.util.Date. (.getTime (->Timestamp date))))) (defn ->Calendar - "Coerce DATE to a `java.util.Calendar`." + "Coerce `date` to a `java.util.Calendar`." (^java.util.Calendar [] (doto (Calendar/getInstance) (.setTimeZone (TimeZone/getTimeZone "UTC")))) @@ -250,7 +256,7 @@ (.setTimeZone (TimeZone/getTimeZone timezone-id))))) (defn relative-date - "Return a new `Timestamp` relative to the current time using a relative date UNIT. + "Return a new Timestamp relative to the current time using a relative date `unit`. (relative-date :year -1) -> #inst 2014-11-12 ..." (^java.sql.Timestamp [unit amount] @@ -275,7 +281,7 @@ :year}) (defn date-extract - "Extract UNIT from DATE. DATE defaults to now. + "Extract `unit` from `date`. `date` defaults to now. (date-extract :year) -> 2015" ([unit] @@ -324,7 +330,7 @@ (format "%d-%02d-01'T'ZZ" year month))) (defn date-trunc - "Truncate DATE to UNIT. DATE defaults to now. + "Truncate `date` to `unit`. `date` defaults to now. (date-trunc :month). ;; -> #inst \"2015-11-01T00:00:00\"" @@ -344,7 +350,7 @@ :year (trunc-with-format "yyyy-01-01'T'ZZ" date timezone-id)))) (defn date-trunc-or-extract - "Apply date bucketing with UNIT to DATE. DATE defaults to now." + "Apply date bucketing with `unit` to `date`. `date` defaults to now." ([unit] (date-trunc-or-extract unit (System/currentTimeMillis) "UTC")) ([unit date] @@ -369,9 +375,25 @@ (recur (/ n divisor) more) (format "%.0f %s" (double n) (name unit))))) +(defn format-microseconds + "Format a time interval in microseconds into something more readable." + ^String [microseconds] + (format-nanoseconds (* 1000.0 microseconds))) + +(defn format-milliseconds + "Format a time interval in milliseconds into something more readable." + ^String [milliseconds] + (format-microseconds (* 1000.0 milliseconds))) + +(defn format-seconds + "Format a time interval in seconds into something more readable." + ^String [seconds] + (format-milliseconds (* 1000.0 seconds))) + +;; TODO - Not sure this belongs in the datetime util namespace (defmacro profile - "Like `clojure.core/time`, but lets you specify a MESSAGE that gets printed with the total time, - and formats the time nicely using `format-nanoseconds`." + "Like `clojure.core/time`, but lets you specify a `message` that gets printed with the total time, and formats the + time nicely using `format-nanoseconds`." {:style/indent 1} ([form] `(profile ~(str form) ~form)) @@ -383,8 +405,8 @@ (format-nanoseconds (- (System/nanoTime) start-time#)))))))) (defn- str->date-time-with-formatters - "Attempt to parse `DATE-STR` using `FORMATTERS`. First successful - parse is returned, or nil" + "Attempt to parse `date-str` using `formatters`. First successful parse is returned, or `nil` if it cannot be + successfully parsed." ([formatters date-str] (str->date-time-with-formatters formatters date-str nil)) ([formatters ^String date-str ^TimeZone tz] @@ -401,9 +423,9 @@ (->DateTimeFormatter "yyyy-MM-dd HH:mm:ss.SSS")) (def ^:private ordered-date-parsers - "When using clj-time.format/parse without a formatter, it tries all default formatters, but not ordered by how - likely the date formatters will succeed. This leads to very slow parsing as many attempts fail before the right one - is found. Using this retains that flexibility but improves performance by trying the most likely ones first" + "When using clj-time.format/parse without a formatter, it tries all default formatters, but not ordered by how likely + the date formatters will succeed. This leads to very slow parsing as many attempts fail before the right one is + found. Using this retains that flexibility but improves performance by trying the most likely ones first" (let [most-likely-default-formatters [:mysql :date-hour-minute-second :date-time :date :basic-date-time :basic-date-time-no-ms :date-time :date-time-no-ms]] @@ -412,7 +434,7 @@ (vals (apply dissoc time/formatters most-likely-default-formatters))))) (defn str->date-time - "Like clj-time.format/parse but uses an ordered list of parsers to be faster. Returns the parsed date or nil if it + "Like clj-time.format/parse but uses an ordered list of parsers to be faster. Returns the parsed date, or `nil` if it was unable to be parsed." (^org.joda.time.DateTime [^String date-str] (str->date-time date-str nil)) @@ -425,8 +447,7 @@ [(time/formatter "HH:mmZ") (time/formatter "HH:mm:SSZ") (time/formatter "HH:mm:SS.SSSZ")]))) (defn str->time - "Parse `TIME-STR` and return a `java.sql.Time` instance. Returns nil - if `TIME-STR` can't be parsed." + "Parse `time-str` and return a `java.sql.Time` instance. Returns `nil` if `time-str` can't be parsed." ([^String date-str] (str->time date-str nil)) ([^String date-str ^TimeZone tz] From 19b08f783bd82d36c051ff38dee4348049aa644f Mon Sep 17 00:00:00 2001 From: Simon Belak Date: Wed, 11 Jul 2018 00:30:02 +0200 Subject: [PATCH 052/116] Fix bad merge --- src/metabase/automagic_dashboards/core.clj | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/metabase/automagic_dashboards/core.clj b/src/metabase/automagic_dashboards/core.clj index dbd71afcf8085..b10e688a7fe88 100644 --- a/src/metabase/automagic_dashboards/core.clj +++ b/src/metabase/automagic_dashboards/core.clj @@ -850,14 +850,13 @@ (s/defn ^:private related "Build a balanced list of related X-rays. General composition of the list is determined for each root type individually via `related-selectors`. That recepie is then filled round-robin style." - [dashboard, rule :- (s/maybe rules/Rule)] - (let [root (-> dashboard :context :root)] - (->> (merge (indepth root rule) - (drilldown-fields dashboard) - (related-entities root)) - (fill-related max-related (related-selectors (-> root :entity type))) - (group-by :selector) - (m/map-vals (partial map :entity))))) + [{:keys [root] :as context}, rule :- (s/maybe rules/Rule)] + (->> (merge (indepth root rule) + (drilldown-fields context) + (related-entities root)) + (fill-related max-related (related-selectors (-> root :entity type))) + (group-by :selector) + (m/map-vals (partial map :entity)))) (defn- filter-referenced-fields "Return a map of fields referenced in filter cluase." From 92c33dfd8a502fc93ebc04cf627b0b91fddf549b Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 10 Jul 2018 15:41:18 -0700 Subject: [PATCH 053/116] Misc lint fixes + table columns setting improvments --- .../src/metabase-lib/lib/queries/StructuredQuery.js | 12 ++++++++++++ frontend/src/metabase/lib/dataset.js | 2 +- .../settings/ChartSettingOrderedFields.jsx | 10 +++------- .../metabase/visualizations/visualizations/Table.jsx | 9 +-------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/frontend/src/metabase-lib/lib/queries/StructuredQuery.js b/frontend/src/metabase-lib/lib/queries/StructuredQuery.js index ab7b4590d0593..04f64ee530981 100644 --- a/frontend/src/metabase-lib/lib/queries/StructuredQuery.js +++ b/frontend/src/metabase-lib/lib/queries/StructuredQuery.js @@ -635,6 +635,18 @@ export default class StructuredQuery extends AtomicQuery { return this._updateQuery(Q.removeExpression, arguments); } + // FIELDS + /** + * Returns dimension options that can appear in the `fields` clause + */ + fieldsOptions(dimensionFilter = () => true): DimensionOptions { + if (this.isBareRows() && this.breakouts().length === 0) { + return this.dimensionOptions(dimensionFilter); + } + // TODO: allow adding fields connected by broken out PKs? + return { count: 0, dimensions: [], fks: [] }; + } + // DIMENSION OPTIONS // TODO Atte Keinänen 6/18/17: Refactor to dimensionOptions which takes a dimensionFilter diff --git a/frontend/src/metabase/lib/dataset.js b/frontend/src/metabase/lib/dataset.js index b6cfe006f37c3..c052353a59bb1 100644 --- a/frontend/src/metabase/lib/dataset.js +++ b/frontend/src/metabase/lib/dataset.js @@ -9,7 +9,7 @@ import type { ColumnName, DatasetData, } from "metabase/meta/types/Dataset"; -import type { Card, VisualizationSettings } from "metabase/meta/types/Card"; +import type { Card } from "metabase/meta/types/Card"; import type { ConcreteField } from "metabase/meta/types/Query"; type ColumnSetting = { diff --git a/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx b/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx index 81ada5a852a9f..e1fdec9cd13b1 100644 --- a/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx +++ b/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx @@ -1,10 +1,7 @@ import React, { Component } from "react"; import { t } from "c-3po"; -import CheckBox from "metabase/components/CheckBox.jsx"; import Icon from "metabase/components/Icon.jsx"; -import PopoverWithTrigger from "metabase/components/PopoverWithTrigger"; -import FieldList from "metabase/query_builder/components/FieldList"; import { SortableContainer, SortableElement } from "react-sortable-hoc"; @@ -15,7 +12,6 @@ import { } from "metabase/lib/dataset"; import { getFriendlyName } from "metabase/visualizations/lib/utils"; -import cx from "classnames"; import _ from "underscore"; const SortableColumn = SortableElement( @@ -74,12 +70,12 @@ export default class ChartSettingOrderedColumns extends Component { ); render() { - const { value, question, addField, columns, columnNames } = this.props; + const { value, question, addField, columns } = this.props; let additionalFieldOptions = { count: 0 }; if (columns && question && question.query() instanceof StructuredQuery) { const fieldRefs = columns.map(column => fieldRefForColumn(column)); - additionalFieldOptions = question.query().dimensionOptions(dimension => { + additionalFieldOptions = question.query().fieldsOptions(dimension => { const mbql = dimension.mbql(); return !_.find(fieldRefs, fieldRef => _.isEqual(fieldRef, mbql)); }); @@ -108,7 +104,7 @@ export default class ChartSettingOrderedColumns extends Component {
)} {disabledColumns.length > 0 || additionalFieldOptions.count > 0 ? ( -

{`More fields`}

+

{`More fields`}

) : null} {disabledColumns.map(columnSetting => ( ({ columns: cols, - columnNames: cols.reduce( - (o, col) => ({ ...o, [col.name]: getFriendlyName(col) }), - {}, - ), }), }, "table.column_widths": {}, From f9f742f4dd4086080e11d3024ed258e97c34417e Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Tue, 10 Jul 2018 17:38:11 -0700 Subject: [PATCH 054/116] spacing and sizing --- frontend/src/metabase/components/BrowseApp.jsx | 2 +- frontend/src/metabase/components/CollectionLanding.jsx | 4 ++-- frontend/src/metabase/components/EntityItem.jsx | 8 ++++---- frontend/src/metabase/components/ItemTypeFilterBar.jsx | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/src/metabase/components/BrowseApp.jsx b/frontend/src/metabase/components/BrowseApp.jsx index f167822407598..cbd086d1207fc 100644 --- a/frontend/src/metabase/components/BrowseApp.jsx +++ b/frontend/src/metabase/components/BrowseApp.jsx @@ -109,7 +109,7 @@ export class TableBrowser extends React.Component { {({ tables, loading, error }) => { return ( - + - + -

{currentCollection.name}

+

{currentCollection.name}

 diff --git a/frontend/src/metabase/components/EntityItem.jsx b/frontend/src/metabase/components/EntityItem.jsx index b77542e0675cd..9c4d9ffaf4a72 100644 --- a/frontend/src/metabase/components/EntityItem.jsx +++ b/frontend/src/metabase/components/EntityItem.jsx @@ -12,7 +12,7 @@ import Icon from "metabase/components/Icon"; import colors from "metabase/lib/colors"; const EntityItemWrapper = Flex.extend` - border-bottom: 1px solid ${colors["bg-light"]}; + border-bottom: 1px solid ${colors["bg-medium"]}; /* TODO - figure out how to use the prop instead of this? */ align-items: center; &:hover { @@ -52,10 +52,10 @@ const EntityItem = ({ ].filter(action => action); return ( - + { const { location } = props; return ( - + {props.filters.map(f => { let isActive = location.query.type === f.filter; From ec62af1d91b3b0dd589972ce3f04b05cb547d7af Mon Sep 17 00:00:00 2001 From: Kyle Doherty Date: Tue, 10 Jul 2018 18:28:44 -0700 Subject: [PATCH 055/116] lint --- frontend/src/metabase/components/CollectionLanding.jsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/metabase/components/CollectionLanding.jsx b/frontend/src/metabase/components/CollectionLanding.jsx index caed143885530..e2e6bbb6c9079 100644 --- a/frontend/src/metabase/components/CollectionLanding.jsx +++ b/frontend/src/metabase/components/CollectionLanding.jsx @@ -430,7 +430,9 @@ class CollectionLanding extends React.Component { ]} /> -

{currentCollection.name}

+

+ {currentCollection.name} +

From 2fe051cd15ae71f1cbb85147ef3b019aaaee4341 Mon Sep 17 00:00:00 2001 From: Kyle Doherty Date: Wed, 11 Jul 2018 11:17:26 -0700 Subject: [PATCH 056/116] fix export and binds --- .../src/metabase/components/CreateDashboardModal.jsx | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/frontend/src/metabase/components/CreateDashboardModal.jsx b/frontend/src/metabase/components/CreateDashboardModal.jsx index 17aa390c1c4af..212ff78328746 100644 --- a/frontend/src/metabase/components/CreateDashboardModal.jsx +++ b/frontend/src/metabase/components/CreateDashboardModal.jsx @@ -22,12 +22,9 @@ const mapDispatchToProps = { @connect(null, mapDispatchToProps) @withRouter -class CreateDashboardModal extends Component { +export default class CreateDashboardModal extends Component { constructor(props, context) { super(props, context); - this.createNewDash = this.createNewDash.bind(this); - this.setDescription = this.setDescription.bind(this); - this.setName = this.setName.bind(this); this.state = { name: null, @@ -56,7 +53,7 @@ class CreateDashboardModal extends Component { this.setState({ description: event.target.value }); }; - async createNewDash(event) { + createNewDash = async event => { event.preventDefault(); let name = this.state.name && this.state.name.trim(); @@ -156,6 +153,3 @@ class CreateDashboardModal extends Component { } } -export default connect(null, mapDispatchToProps)( - withRouter(CreateDashboardModal), -); From c5b3d542ec8083609eb0f1e0a2eae1d3cf469157 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Wed, 11 Jul 2018 11:39:46 -0700 Subject: [PATCH 057/116] Rename ChartSettingOrdered{Fields,Columns}, fix up unit tests --- frontend/src/metabase/lib/dataset.js | 20 ++++- ...lds.jsx => ChartSettingOrderedColumns.jsx} | 33 ++++++-- .../visualizations/visualizations/Table.jsx | 16 ++-- .../components/ChartSettings.unit.spec.js | 81 ------------------ .../ChartSettingOrderedColumns.unit.spec.js | 84 +++++++++++++++++++ .../ChartSettingOrderedFields.unit.spec.js | 79 ----------------- 6 files changed, 136 insertions(+), 177 deletions(-) rename frontend/src/metabase/visualizations/components/settings/{ChartSettingOrderedFields.jsx => ChartSettingOrderedColumns.jsx} (83%) delete mode 100644 frontend/test/visualizations/components/ChartSettings.unit.spec.js create mode 100644 frontend/test/visualizations/components/settings/ChartSettingOrderedColumns.unit.spec.js delete mode 100644 frontend/test/visualizations/components/settings/ChartSettingOrderedFields.unit.spec.js diff --git a/frontend/src/metabase/lib/dataset.js b/frontend/src/metabase/lib/dataset.js index c052353a59bb1..805ba276894d7 100644 --- a/frontend/src/metabase/lib/dataset.js +++ b/frontend/src/metabase/lib/dataset.js @@ -71,18 +71,30 @@ export function findColumnForColumnSetting( columns: Column[], columnSetting: ColumnSetting, ): ?Column { + const index = findColumnIndexForColumnSetting(columns, columnSetting); + if (index >= 0) { + return columns[index]; + } else { + return null; + } +} + +export function findColumnIndexForColumnSetting( + columns: Column[], + columnSetting: ColumnSetting, +): number { const { fieldRef } = columnSetting; // first try to find by fieldRef if (fieldRef != null) { - const column = _.find(columns, col => + const index = _.findIndex(columns, col => _.isEqual(fieldRef, fieldRefForColumn(col)), ); - if (column) { - return column; + if (index >= 0) { + return index; } } // if that fails, find by column name - return _.findWhere(columns, { name: columnSetting.name }); + return _.findIndex(columns, col => col.name === columnSetting.name); } /** diff --git a/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx b/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedColumns.jsx similarity index 83% rename from frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx rename to frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedColumns.jsx index e1fdec9cd13b1..2e4d4380117eb 100644 --- a/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx +++ b/frontend/src/metabase/visualizations/components/settings/ChartSettingOrderedColumns.jsx @@ -62,6 +62,18 @@ export default class ChartSettingOrderedColumns extends Component { this.props.onChange(fields); }; + handleAddNewField = fieldRef => { + const { value, onChange, addField } = this.props; + onChange([ + // remove duplicates + ...value.filter( + columnSetting => !_.isEqual(columnSetting.fieldRef, fieldRef), + ), + { fieldRef, enabled: true }, + ]); + addField(fieldRef); + }; + getColumnName = columnSetting => getFriendlyName( findColumnForColumnSetting(this.props.columns, columnSetting) || { @@ -70,7 +82,7 @@ export default class ChartSettingOrderedColumns extends Component { ); render() { - const { value, question, addField, columns } = this.props; + const { value, question, columns } = this.props; let additionalFieldOptions = { count: 0 }; if (columns && question && question.query() instanceof StructuredQuery) { @@ -82,7 +94,11 @@ export default class ChartSettingOrderedColumns extends Component { } const [enabledColumns, disabledColumns] = _.partition( - value.map((columnSetting, index) => ({ ...columnSetting, index })), + value + .filter(columnSetting => + findColumnForColumnSetting(columns, columnSetting), + ) + .map((columnSetting, index) => ({ ...columnSetting, index })), columnSetting => columnSetting.enabled, ); @@ -106,18 +122,20 @@ export default class ChartSettingOrderedColumns extends Component { {disabledColumns.length > 0 || additionalFieldOptions.count > 0 ? (

{`More fields`}

) : null} - {disabledColumns.map(columnSetting => ( + {disabledColumns.map((columnSetting, index) => ( this.handleEnable(columnSetting)} /> ))} {additionalFieldOptions.count > 0 && (
- {additionalFieldOptions.dimensions.map(dimension => ( + {additionalFieldOptions.dimensions.map((dimension, index) => ( addField(dimension.mbql())} + onAdd={() => this.handleAddNewField(dimension.mbql())} /> ))} {additionalFieldOptions.fks.map(fk => ( @@ -125,10 +143,11 @@ export default class ChartSettingOrderedColumns extends Component {
{fk.field.target.table.display_name}
- {fk.dimensions.map(dimension => ( + {fk.dimensions.map((dimension, index) => ( addField(dimension.mbql())} + onAdd={() => this.handleAddNewField(dimension.mbql())} /> ))}
diff --git a/frontend/src/metabase/visualizations/visualizations/Table.jsx b/frontend/src/metabase/visualizations/visualizations/Table.jsx index 9c65c562d514b..74701ba5f78c8 100644 --- a/frontend/src/metabase/visualizations/visualizations/Table.jsx +++ b/frontend/src/metabase/visualizations/visualizations/Table.jsx @@ -6,11 +6,12 @@ import TableInteractive from "../components/TableInteractive.jsx"; import TableSimple from "../components/TableSimple.jsx"; import { t } from "c-3po"; import * as DataGrid from "metabase/lib/data_grid"; +import { findColumnIndexForColumnSetting } from "metabase/lib/dataset"; import Query from "metabase/lib/query"; import { isMetric, isDimension } from "metabase/lib/schema_metadata"; import { columnsAreValid } from "metabase/visualizations/lib/utils"; -import ChartSettingOrderedFields from "metabase/visualizations/components/settings/ChartSettingOrderedFields.jsx"; +import ChartSettingOrderedColumns from "metabase/visualizations/components/settings/ChartSettingOrderedColumns.jsx"; import ChartSettingsTableFormatting, { isFormattable, } from "metabase/visualizations/components/settings/ChartSettingsTableFormatting.jsx"; @@ -184,7 +185,7 @@ export default class Table extends Component { "table.columns": { section: "Data", title: t`Visible fields`, - widget: ChartSettingOrderedFields, + widget: ChartSettingOrderedColumns, getHidden: (series, vizSettings) => vizSettings["table.pivot"], isValid: ([{ card, data }]) => card.visualization_settings["table.columns"] && @@ -298,10 +299,13 @@ export default class Table extends Component { }); } else { const { cols, rows, columns } = data; - const columnIndexes = settings["table.columns"] - .filter(f => f.enabled) - .map(f => _.findIndex(cols, c => c.name === f.name)) - .filter(i => i >= 0 && i < cols.length); + const columnSettings = settings["table.columns"]; + const columnIndexes = columnSettings + .filter(columnSetting => columnSetting.enabled) + .map(columnSetting => + findColumnIndexForColumnSetting(cols, columnSetting), + ) + .filter(columnIndex => columnIndex >= 0 && columnIndex < cols.length); this.setState({ data: { diff --git a/frontend/test/visualizations/components/ChartSettings.unit.spec.js b/frontend/test/visualizations/components/ChartSettings.unit.spec.js deleted file mode 100644 index 1729473d62e72..0000000000000 --- a/frontend/test/visualizations/components/ChartSettings.unit.spec.js +++ /dev/null @@ -1,81 +0,0 @@ -import React from "react"; - -import ChartSettings from "metabase/visualizations/components/ChartSettings"; - -import { TableCard } from "../__support__/visualizations"; - -import { mount } from "enzyme"; -import { click } from "__support__/enzyme_utils"; - -function renderChartSettings(enabled = true) { - const props = { - series: [ - TableCard("Foo", { - card: { - visualization_settings: { - "table.columns": [{ name: "Foo_col0", enabled: enabled }], - }, - }, - }), - ], - }; - return mount( {}} />); -} - -// The ExplicitSize component uses the matchMedia DOM API -// which does not exist in jest's JSDOM -Object.defineProperty(window, "matchMedia", { - value: jest.fn(() => { - return { - matches: true, - addListener: () => {}, - removeListener: () => {}, - }; - }), -}); - -// We have to do some mocking here to avoid calls to GA and to Metabase settings -jest.mock("metabase/lib/settings", () => ({ - get: () => "v", -})); - -describe("ChartSettings", () => { - describe("toggling fields", () => { - describe("disabling all fields", () => { - it("should show null state", () => { - const chartSettings = renderChartSettings(); - - expect(chartSettings.find(".toggle-all .Icon-check").length).toEqual(1); - expect(chartSettings.find("table").length).toEqual(1); - - click(chartSettings.find(".toggle-all .cursor-pointer")); - - expect(chartSettings.find(".toggle-all .Icon-check").length).toEqual(0); - expect(chartSettings.find("table").length).toEqual(0); - expect(chartSettings.text()).toContain( - "Every field is hidden right now", - ); - }); - }); - - describe("enabling all fields", () => { - it("should show all columns", () => { - const chartSettings = renderChartSettings(false); - - expect(chartSettings.find(".toggle-all .Icon-check").length).toEqual(0); - expect(chartSettings.find("table").length).toEqual(0); - expect(chartSettings.text()).toContain( - "Every field is hidden right now", - ); - - click(chartSettings.find(".toggle-all .cursor-pointer")); - - expect(chartSettings.find(".toggle-all .Icon-check").length).toEqual(1); - expect(chartSettings.find("table").length).toEqual(1); - expect(chartSettings.text()).not.toContain( - "Every field is hidden right now", - ); - }); - }); - }); -}); diff --git a/frontend/test/visualizations/components/settings/ChartSettingOrderedColumns.unit.spec.js b/frontend/test/visualizations/components/settings/ChartSettingOrderedColumns.unit.spec.js new file mode 100644 index 0000000000000..b2cb958d61e38 --- /dev/null +++ b/frontend/test/visualizations/components/settings/ChartSettingOrderedColumns.unit.spec.js @@ -0,0 +1,84 @@ +import React from "react"; + +import ChartSettingOrderedColumns from "metabase/visualizations/components/settings/ChartSettingOrderedColumns"; + +import { mount } from "enzyme"; + +import { question } from "../../../__support__/sample_dataset_fixture.js"; + +function renderChartSettingOrderedColumns(props) { + return mount( + {}} + columns={[{ name: "Foo" }, { name: "Bar" }]} + {...props} + />, + ); +} + +describe("ChartSettingOrderedColumns", () => { + it("should have the correct add and remove buttons", () => { + const setting = renderChartSettingOrderedColumns({ + value: [{ name: "Foo", enabled: true }, { name: "Bar", enabled: false }], + }); + expect(setting.find(".Icon-add")).toHaveLength(1); + expect(setting.find(".Icon-close")).toHaveLength(1); + }); + it("should add a column", () => { + const onChange = jest.fn(); + const setting = renderChartSettingOrderedColumns({ + value: [{ name: "Foo", enabled: true }, { name: "Bar", enabled: false }], + onChange, + }); + setting.find(".Icon-add").simulate("click"); + expect(onChange.mock.calls).toEqual([ + [[{ name: "Foo", enabled: true }, { name: "Bar", enabled: true }]], + ]); + }); + it("should remove a column", () => { + const onChange = jest.fn(); + const setting = renderChartSettingOrderedColumns({ + value: [{ name: "Foo", enabled: true }, { name: "Bar", enabled: false }], + onChange, + }); + setting.find(".Icon-close").simulate("click"); + expect(onChange.mock.calls).toEqual([ + [[{ name: "Foo", enabled: false }, { name: "Bar", enabled: false }]], + ]); + }); + it("should reorder columns", () => { + const onChange = jest.fn(); + const setting = renderChartSettingOrderedColumns({ + value: [{ name: "Foo", enabled: true }, { name: "Bar", enabled: true }], + onChange, + }); + // just call handleSortEnd directly for now as it's difficult to simulate drag and drop + setting.instance().handleSortEnd({ oldIndex: 1, newIndex: 0 }); + expect(onChange.mock.calls).toEqual([ + [[{ name: "Bar", enabled: true }, { name: "Foo", enabled: true }]], + ]); + }); + + describe("for structured queries", () => { + it("should list and add additional columns", () => { + const onChange = jest.fn(); + const addField = jest.fn(); + const setting = renderChartSettingOrderedColumns({ + value: [], + columns: [], + question, + onChange, + addField, + }); + expect(setting.find(".Icon-add")).toHaveLength(28); + setting + .find(".Icon-add") + .first() + .simulate("click"); + expect(addField.mock.calls).toEqual([[["field-id", 1]]]); + expect(onChange.mock.calls).toEqual([ + [[{ fieldRef: ["field-id", 1], enabled: true }]], + ]); + }); + }); +}); diff --git a/frontend/test/visualizations/components/settings/ChartSettingOrderedFields.unit.spec.js b/frontend/test/visualizations/components/settings/ChartSettingOrderedFields.unit.spec.js deleted file mode 100644 index 84cc84d4e8208..0000000000000 --- a/frontend/test/visualizations/components/settings/ChartSettingOrderedFields.unit.spec.js +++ /dev/null @@ -1,79 +0,0 @@ -import React from "react"; - -import ChartSettingOrderedFields from "metabase/visualizations/components/settings/ChartSettingOrderedFields"; - -import { mount } from "enzyme"; - -function renderChartSettingOrderedFields(props) { - return mount( {}} {...props} />); -} - -describe("ChartSettingOrderedFields", () => { - describe("isAnySelected", () => { - describe("when on or more fields are enabled", () => { - it("should be true", () => { - const chartSettings = renderChartSettingOrderedFields({ - columnNames: { id: "ID", text: "Text" }, - value: [ - { name: "id", enabled: true }, - { name: "text", enabled: false }, - ], - }); - expect(chartSettings.instance().isAnySelected()).toEqual(true); - }); - }); - - describe("when no fields are enabled", () => { - it("should be false", () => { - const chartSettings = renderChartSettingOrderedFields({ - columnNames: { id: "ID", text: "Text" }, - value: [ - { name: "id", enabled: false }, - { name: "text", enabled: false }, - ], - }); - expect(chartSettings.instance().isAnySelected()).toEqual(false); - }); - }); - }); - - describe("toggleAll", () => { - describe("when passed false", () => { - it("should mark all fields as enabled", () => { - const onChange = jest.fn(); - const chartSettings = renderChartSettingOrderedFields({ - columnNames: { id: "ID", text: "Text" }, - value: [ - { name: "id", enabled: false }, - { name: "text", enabled: false }, - ], - onChange, - }); - chartSettings.instance().handleToggleAll(false); - expect(onChange.mock.calls[0][0]).toEqual([ - { name: "id", enabled: true }, - { name: "text", enabled: true }, - ]); - }); - }); - - describe("when passed true", () => { - it("should mark all fields as disabled", () => { - const onChange = jest.fn(); - const chartSettings = renderChartSettingOrderedFields({ - columnNames: { id: "ID", text: "Text" }, - value: [ - { name: "id", enabled: true }, - { name: "text", enabled: true }, - ], - onChange, - }); - chartSettings.instance().handleToggleAll(true); - expect(onChange.mock.calls[0][0]).toEqual([ - { name: "id", enabled: false }, - { name: "text", enabled: false }, - ]); - }); - }); - }); -}); From 410b5dd32ed96449dc6107ec53931dfb2b9d81ea Mon Sep 17 00:00:00 2001 From: Cam Saul Date: Wed, 11 Jul 2018 12:23:04 -0700 Subject: [PATCH 058/116] Tests for new defsetting options --- test/metabase/models/setting_test.clj | 60 +++++++++++++++++++++++++++ test/metabase/test/util.clj | 5 ++- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/test/metabase/models/setting_test.clj b/test/metabase/models/setting_test.clj index c7c55ef06ac6b..6474e20c67236 100644 --- a/test/metabase/models/setting_test.clj +++ b/test/metabase/models/setting_test.clj @@ -49,6 +49,10 @@ (test-setting-2 setting-2-value)) +(expect + String + (:tag (meta #'test-setting-1))) + ;; ## GETTERS ;; Test defsetting getter fn. Should return the value from env var MB_TEST_SETTING_1 (expect "ABCDEFG" @@ -219,6 +223,10 @@ ;;; ------------------------------------------------ BOOLEAN SETTINGS ------------------------------------------------ +(expect + Boolean + (:tag (meta #'test-boolean-setting))) + (expect {:value nil, :is_env_setting false, :env_name "MB_TEST_BOOLEAN_SETTING", :default nil} (user-facing-info-with-db-and-env-var-values :test-boolean-setting nil nil)) @@ -293,6 +301,10 @@ ;; ok, make sure the setting was set (toucan-name))) +(expect + String + (:tag (meta #'toucan-name))) + ;;; ----------------------------------------------- Encrypted Settings ----------------------------------------------- @@ -323,6 +335,23 @@ (actual-value-in-db :toucan-name))) +;;; ----------------------------------------------- TIMESTAMP SETTINGS ----------------------------------------------- + +(defsetting ^:private test-timestamp-setting + "Test timestamp setting" + :type :timestamp) + +(expect + java.sql.Timestamp + (:tag (meta #'test-timestamp-setting))) + +;; make sure we can set & fetch the value and that it gets serialized/deserialized correctly +(expect + #inst "2018-07-11T09:32:00.000Z" + (do (test-timestamp-setting #inst "2018-07-11T09:32:00.000Z") + (test-timestamp-setting))) + + ;;; --------------------------------------------- Cache Synchronization ---------------------------------------------- (def ^:private settings-last-updated-key @(resolve 'metabase.models.setting/settings-last-updated-key)) @@ -444,3 +473,34 @@ ;; detect a cache out-of-date situation and flush the cache as appropriate, giving us the updated value when we ;; call! :wow: (toucan-name))) + + +;;; ----------------------------------------------- Uncached Settings ------------------------------------------------ + +(defsetting ^:private uncached-setting + "A test setting that should *not* be cached." + :cache? false) + +;; make sure uncached setting still saves to the DB +(expect + "ABCDEF" + (encryption-test/with-secret-key nil + (uncached-setting "ABCDEF") + (actual-value-in-db "uncached-setting"))) + +;; make sure that fetching the Setting always fetches the latest value from the DB +(expect + "123456" + (encryption-test/with-secret-key nil + (uncached-setting "ABCDEF") + (db/update-where! Setting {:key "uncached-setting"} + :value "123456") + (uncached-setting))) + +;; make sure that updating the setting doesn't update the last-updated timestamp in the cache $$ +(expect + nil + (encryption-test/with-secret-key nil + (clear-settings-last-updated-value-in-db!) + (uncached-setting "abcdef") + (settings-last-updated-value-in-db))) diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj index 006e5cf54a798..304763012af27 100644 --- a/test/metabase/test/util.clj +++ b/test/metabase/test/util.clj @@ -37,7 +37,8 @@ [dataset-definitions :as defs]] [toucan.db :as db] [toucan.util.test :as test]) - (:import java.util.TimeZone + (:import com.mchange.v2.c3p0.PooledDataSource + java.util.TimeZone org.joda.time.DateTimeZone [org.quartz CronTrigger JobDetail JobKey Scheduler Trigger])) @@ -451,7 +452,7 @@ timezone. That causes problems for tests that we can determine the database's timezone. This function will reset the connections in the connection pool for `db` to ensure that we get fresh session with no timezone specified" [db] - (when-let [conn-pool (:datasource (sql/db->pooled-connection-spec db))] + (when-let [^PooledDataSource conn-pool (:datasource (sql/db->pooled-connection-spec db))] (.softResetAllUsers conn-pool))) (defn db-timezone-id From e8832ade8ed3aefddcdafe50eeda24570df636f6 Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Wed, 11 Jul 2018 12:28:25 -0700 Subject: [PATCH 059/116] tweak cards and browse all link --- frontend/src/metabase/components/Card.jsx | 7 ++++--- .../src/metabase/components/CollectionLanding.jsx | 6 +++--- frontend/src/metabase/components/EntityItem.jsx | 14 ++++++++++++-- frontend/src/metabase/containers/Overworld.jsx | 4 ++-- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/frontend/src/metabase/components/Card.jsx b/frontend/src/metabase/components/Card.jsx index e6072c9ac1379..6c0d6f4084e8b 100644 --- a/frontend/src/metabase/components/Card.jsx +++ b/frontend/src/metabase/components/Card.jsx @@ -5,14 +5,15 @@ import colors, { alpha } from "metabase/lib/colors"; const Card = styled.div` ${space} background-color: ${props => props.dark ? colors["text-dark"] : "white"}; - border: 1px solid ${props => (props.dark ? "transparent" : colors["border"])}; + border: 1px solid ${props => (props.dark ? "transparent" : colors["bg-medium"])}; ${props => props.dark && `color: white`}; border-radius: 6px; - box-shadow: 0 5px 22px ${props => colors["shadow"]}; + box-shadow: 0 7px 20px ${props => colors["shadow"]}; + transition: all 0.2s linear; ${props => props.hoverable && `&:hover { - box-shadow: 0 5px 16px ${alpha(colors["shadow"], 0.1)}; + box-shadow: 0 10px 22px ${alpha(colors["shadow"], 0.09)}; }`}; `; diff --git a/frontend/src/metabase/components/CollectionLanding.jsx b/frontend/src/metabase/components/CollectionLanding.jsx index caed143885530..6a182d92e07f6 100644 --- a/frontend/src/metabase/components/CollectionLanding.jsx +++ b/frontend/src/metabase/components/CollectionLanding.jsx @@ -15,7 +15,7 @@ import Button from "metabase/components/Button"; import Card from "metabase/components/Card"; import Modal from "metabase/components/Modal"; import StackedCheckBox from "metabase/components/StackedCheckBox"; -import EntityItem from "metabase/components/EntityItem"; +import EntityListItem from "metabase/components/EntityItem"; import { Grid, GridItem } from "metabase/components/Grid"; import Icon from "metabase/components/Icon"; import Link from "metabase/components/Link"; @@ -309,7 +309,7 @@ export const NormalItem = ({ onMove, }) => ( - 0} selectable item={item} @@ -414,7 +414,7 @@ class CollectionLanding extends React.Component { return ( - + + + + + +export const EntityCardItem = (props) => + + + + const EntityItem = ({ name, iconName, @@ -52,7 +62,7 @@ const EntityItem = ({ ].filter(action => action); return ( - + - + -

{t`Browse all items`}

+

{t`Browse all items`}

 From 31c426557b47ecd9df5918619c0bbc10670bae64 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Wed, 11 Jul 2018 12:39:37 -0700 Subject: [PATCH 060/116] Collection picker handle personal collections --- .../src/metabase/containers/ItemPicker.jsx | 24 +++++--- frontend/src/metabase/entities/collections.js | 57 +++++++++++++++++-- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/frontend/src/metabase/containers/ItemPicker.jsx b/frontend/src/metabase/containers/ItemPicker.jsx index 9a8ae829f8e77..45c05002cae24 100644 --- a/frontend/src/metabase/containers/ItemPicker.jsx +++ b/frontend/src/metabase/containers/ItemPicker.jsx @@ -6,17 +6,21 @@ import { Flex, Box } from "grid-styled"; import Icon from "metabase/components/Icon"; import Breadcrumbs from "metabase/components/Breadcrumbs"; +import { connect } from "react-redux"; import EntityListLoader, { entityListLoader, } from "metabase/entities/containers/EntityListLoader"; -import { getExpandedCollectionsById } from "metabase/entities/collections"; +import Collections from "metabase/entities/collections"; const COLLECTION_ICON_COLOR = "#DCE1E4"; const isRoot = collection => collection.id === "root" || collection.id == null; @entityListLoader({ entityType: "collections" }) +@connect((state, props) => ({ + collectionsById: Collections.selectors.getExpandedCollectionsById(state), +})) export default class ItemPicker extends React.Component { constructor(props) { super(props); @@ -55,18 +59,13 @@ export default class ItemPicker extends React.Component { } render() { - const { value, onChange, collections, style, className } = this.props; + const { value, onChange, collectionsById, style, className } = this.props; const { parentId } = this.state; const models = new Set(this.props.models); const modelsIncludeNonCollections = this.props.models.filter(model => model !== "collection").length > 0; - if (!collections) { - return
nope
; - } - - const collectionsById = getExpandedCollectionsById(collections); const collection = collectionsById[parentId]; const crumbs = this._getCrumbs(collection, collectionsById); @@ -83,10 +82,15 @@ export default class ItemPicker extends React.Component { model: "collection", })); + // special case for root collection + const getId = item => + item && + (item.model === "collection" && item.id === null ? "root" : item.id); + const isSelected = item => item && value && - item.id === value.id && + getId(item) === getId(value) && (models.size === 1 || item.model === value.model); return ( @@ -102,7 +106,9 @@ export default class ItemPicker extends React.Component { color={COLLECTION_ICON_COLOR} icon="all" selected={isSelected(collection) && models.has("collection")} - canSelect={models.has("collection")} + canSelect={ + models.has("collection") && collection.can_edit !== false + } hasChildren={ (collection.children && collection.children.length > 0 && diff --git a/frontend/src/metabase/entities/collections.js b/frontend/src/metabase/entities/collections.js index ae38c1c0031bc..5a3e69bc4e430 100644 --- a/frontend/src/metabase/entities/collections.js +++ b/frontend/src/metabase/entities/collections.js @@ -5,6 +5,8 @@ import colors from "metabase/lib/colors"; import { CollectionSchema } from "metabase/schema"; import { createSelector } from "reselect"; +import { getUser } from "metabase/selectors/user"; + import { t } from "c-3po"; const Collections = createEntity({ @@ -30,8 +32,12 @@ const Collections = createEntity({ selectors: { getExpandedCollectionsById: createSelector( - [state => state.entities.collections], - collections => getExpandedCollectionsById(Object.values(collections)), + [state => state.entities.collections, getUser], + (collections, user) => + getExpandedCollectionsById( + Object.values(collections), + user && user.personal_collection_id, + ), ), }, @@ -88,15 +94,33 @@ export const canonicalCollectionId = collectionId => export const ROOT_COLLECTION = { id: "root", - name: "Our analytics", + name: t`Our analytics`, location: "", path: [], }; +export const PERSONAL_COLLECTIONS = { + id: "personal", + name: t`Personal Collections`, + location: "/", + path: ["root"], + can_edit: false, +}; + // given list of collections with { id, name, location } returns a map of ids to // expanded collection objects like { id, name, location, path, children } // including a root collection -export function getExpandedCollectionsById(collections) { +function getExpandedCollectionsById( + collections, + userPersonalCollectionId, + includePersonalCollections = true, +) { + const personalCollections = collections.filter( + collection => collection.personal_owner_id != null, + ); + collections = collections.filter( + collection => collection.personal_owner_id == null, + ); const collectionsById = {}; for (const c of collections) { collectionsById[c.id] = { @@ -118,6 +142,31 @@ export function getExpandedCollectionsById(collections) { ...(collectionsById[ROOT_COLLECTION.id] || {}), }; + // "My personal collection" + if (userPersonalCollectionId != null) { + collectionsById[ROOT_COLLECTION.id].children.push({ + id: userPersonalCollectionId, + name: t`My personal collection`, + location: "/", + path: ["root"], + parent: collectionsById[PERSONAL_COLLECTIONS.id], + }); + } + + // "Personal Collections" + if (includePersonalCollections && personalCollections.length > 0) { + collectionsById[PERSONAL_COLLECTIONS.id] = { + children: personalCollections, + ...PERSONAL_COLLECTIONS, + }; + collectionsById[ROOT_COLLECTION.id].children.push( + collectionsById[PERSONAL_COLLECTIONS.id], + ); + for (const c of personalCollections) { + c.parent = collectionsById[PERSONAL_COLLECTIONS.id]; + } + } + // iterate over original collections so we don't include ROOT_COLLECTION as // a child of itself for (const { id } of collections) { From dcc4870d62a691dbc2cb5d6b651d06402f5a490e Mon Sep 17 00:00:00 2001 From: Kyle Doherty Date: Wed, 11 Jul 2018 12:58:34 -0700 Subject: [PATCH 061/116] initial fixes (#8033) --- frontend/src/metabase/nav/containers/Navbar.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/metabase/nav/containers/Navbar.jsx b/frontend/src/metabase/nav/containers/Navbar.jsx index 0a06e8ce43a77..cb52ea0b82e5b 100644 --- a/frontend/src/metabase/nav/containers/Navbar.jsx +++ b/frontend/src/metabase/nav/containers/Navbar.jsx @@ -270,10 +270,11 @@ export default class Navbar extends Component { - + Date: Wed, 11 Jul 2018 13:07:17 -0700 Subject: [PATCH 062/116] lint --- frontend/src/metabase/components/Card.jsx | 3 ++- .../src/metabase/components/EntityItem.jsx | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/frontend/src/metabase/components/Card.jsx b/frontend/src/metabase/components/Card.jsx index 6c0d6f4084e8b..faf07b1cffbd3 100644 --- a/frontend/src/metabase/components/Card.jsx +++ b/frontend/src/metabase/components/Card.jsx @@ -5,7 +5,8 @@ import colors, { alpha } from "metabase/lib/colors"; const Card = styled.div` ${space} background-color: ${props => props.dark ? colors["text-dark"] : "white"}; - border: 1px solid ${props => (props.dark ? "transparent" : colors["bg-medium"])}; + border: 1px solid + ${props => (props.dark ? "transparent" : colors["bg-medium"])}; ${props => props.dark && `color: white`}; border-radius: 6px; box-shadow: 0 7px 20px ${props => colors["shadow"]}; diff --git a/frontend/src/metabase/components/EntityItem.jsx b/frontend/src/metabase/components/EntityItem.jsx index b822a38255709..33f8094303125 100644 --- a/frontend/src/metabase/components/EntityItem.jsx +++ b/frontend/src/metabase/components/EntityItem.jsx @@ -20,15 +20,17 @@ const EntityItemWrapper = Flex.extend` } `; -export const EntityListItem = (props) => - - - +export const EntityListItem = props => ( + + + +); -export const EntityCardItem = (props) => - - - +export const EntityCardItem = props => ( + + + +); const EntityItem = ({ name, From 565082db656527cb483bcc713620dfe5ff428f1b Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Wed, 11 Jul 2018 13:39:32 -0700 Subject: [PATCH 063/116] Basic item picker search, fix personal collections children --- .../src/metabase/containers/ItemPicker.jsx | 101 ++++++++++++------ frontend/src/metabase/entities/collections.js | 73 +++++++------ 2 files changed, 111 insertions(+), 63 deletions(-) diff --git a/frontend/src/metabase/containers/ItemPicker.jsx b/frontend/src/metabase/containers/ItemPicker.jsx index 45c05002cae24..22ec5b9890464 100644 --- a/frontend/src/metabase/containers/ItemPicker.jsx +++ b/frontend/src/metabase/containers/ItemPicker.jsx @@ -26,6 +26,8 @@ export default class ItemPicker extends React.Component { super(props); this.state = { parentId: "root", + searchMode: false, + searchString: false, }; } @@ -60,7 +62,7 @@ export default class ItemPicker extends React.Component { render() { const { value, onChange, collectionsById, style, className } = this.props; - const { parentId } = this.state; + const { parentId, searchMode, searchString } = this.state; const models = new Set(this.props.models); const modelsIncludeNonCollections = @@ -95,41 +97,73 @@ export default class ItemPicker extends React.Component { return ( - - - - - {allCollections.map(collection => ( - 0 && - // exclude root since we show root's subcollections alongside it - !isRoot(collection)) || - modelsIncludeNonCollections - } - onChange={collection => - isRoot(collection) - ? // "root" collection should have `null` id - onChange({ id: null, model: "collection" }) - : onChange(collection) + {searchMode ? ( + + { + if (e.key === "Enter") { + this.setState({ searchString: e.target.value }); + } + }} + /> + + this.setState({ searchMode: null, searchString: null }) } - onChangeParentId={parentId => this.setState({ parentId })} /> - ))} - {modelsIncludeNonCollections && ( + + ) : ( + + + this.setState({ searchMode: true })} + /> + + )} + + {!searchString + ? allCollections.map(collection => ( + 0 && + // exclude root since we show root's subcollections alongside it + !isRoot(collection)) || + modelsIncludeNonCollections + } + onChange={collection => + isRoot(collection) + ? // "root" collection should have `null` id + onChange({ id: null, model: "collection" }) + : onChange(collection) + } + onChangeParentId={parentId => this.setState({ parentId })} + /> + )) + : null} + {(modelsIncludeNonCollections || searchString) && ( - item.model !== "collection" && models.has(item.model), + // remove collections unless we're searching + (item.model !== "collection" || !!searchString) && + // only include desired models (TODO: ideally the endpoint would handle this) + models.has(item.model), ) .map(item => ( collection.personal_owner_id != null, - ); - collections = collections.filter( - collection => collection.personal_owner_id == null, - ); +function getExpandedCollectionsById(collections, userPersonalCollectionId) { const collectionsById = {}; for (const c of collections) { collectionsById[c.id] = { @@ -135,44 +135,49 @@ function getExpandedCollectionsById( }; } - // make sure we have the root collection with all relevant info + // "Our Analytics" collectionsById[ROOT_COLLECTION.id] = { - children: [], ...ROOT_COLLECTION, + parent: null, + children: [], ...(collectionsById[ROOT_COLLECTION.id] || {}), }; // "My personal collection" if (userPersonalCollectionId != null) { collectionsById[ROOT_COLLECTION.id].children.push({ + ...PERSONAL_COLLECTION, id: userPersonalCollectionId, - name: t`My personal collection`, - location: "/", - path: ["root"], - parent: collectionsById[PERSONAL_COLLECTIONS.id], + parent: collectionsById[ROOT_COLLECTION.id], + children: [], }); } // "Personal Collections" - if (includePersonalCollections && personalCollections.length > 0) { - collectionsById[PERSONAL_COLLECTIONS.id] = { - children: personalCollections, - ...PERSONAL_COLLECTIONS, - }; - collectionsById[ROOT_COLLECTION.id].children.push( - collectionsById[PERSONAL_COLLECTIONS.id], - ); - for (const c of personalCollections) { - c.parent = collectionsById[PERSONAL_COLLECTIONS.id]; - } - } + collectionsById[PERSONAL_COLLECTIONS.id] = { + ...PERSONAL_COLLECTIONS, + parent: collectionsById[ROOT_COLLECTION.id], + children: [], + }; + collectionsById[ROOT_COLLECTION.id].children.push( + collectionsById[PERSONAL_COLLECTIONS.id], + ); // iterate over original collections so we don't include ROOT_COLLECTION as // a child of itself for (const { id } of collections) { const c = collectionsById[id]; if (c.path) { - const parent = c.path[c.path.length - 1] || "root"; + let parent; + // move personal collections into PERSONAL_COLLECTIONS fake collection + if (c.personal_owner_id != null) { + parent = PERSONAL_COLLECTIONS.id; + } else if (c.path[c.path.length - 1]) { + parent = c.path[c.path.length - 1]; + } else { + parent = ROOT_COLLECTION.id; + } + c.parent = collectionsById[parent]; // need to ensure the parent collection exists, it may have been filtered // because we're selecting a collection's parent collection and it can't @@ -182,5 +187,11 @@ function getExpandedCollectionsById( } } } + + // remove PERSONAL_COLLECTIONS collection if there are none + if (collectionsById[PERSONAL_COLLECTIONS.id].children.length === 0) { + delete collectionsById[PERSONAL_COLLECTIONS.id]; + } + return collectionsById; } From 0354580cc34084c1c1c87838fdade74e226d851b Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Wed, 11 Jul 2018 13:54:02 -0700 Subject: [PATCH 064/116] Add inheritWidth prop to ItemSelect, default true --- .../src/metabase/containers/ItemSelect.jsx | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/frontend/src/metabase/containers/ItemSelect.jsx b/frontend/src/metabase/containers/ItemSelect.jsx index 8e889a805eb5e..6a4ed4bc310cf 100644 --- a/frontend/src/metabase/containers/ItemSelect.jsx +++ b/frontend/src/metabase/containers/ItemSelect.jsx @@ -1,11 +1,18 @@ import React from "react"; +import ReactDOM from "react-dom"; import PropTypes from "prop-types"; import PopoverWithTrigger from "metabase/components/PopoverWithTrigger"; import SelectButton from "metabase/components/SelectButton"; +const MIN_POPOVER_WIDTH = 300; + export default (PickerComponent, NameComponent, type) => class ItemSelect extends React.Component { + state = { + width: null, + }; + static propTypes = { // collection ID, null (for root collection), or undefined value: PropTypes.number, @@ -13,12 +20,23 @@ export default (PickerComponent, NameComponent, type) => field: PropTypes.object.isRequired, // optional collectionId to filter out so you can't move a collection into itself collectionId: PropTypes.number, + // make the popover content inherit the select widget's width + inheritWidth: PropTypes.bool, }; static defaultProps = { placeholder: `Select a ${type}`, + inheritWidth: true, }; + componentDidUpdate() { + // save the width so we can make the poopver content match + const width = ReactDOM.findDOMNode(this).clientWidth; + if (this.state.width !== width) { + this.setState({ width }); + } + } + render() { const { value, @@ -26,6 +44,7 @@ export default (PickerComponent, NameComponent, type) => className, style, placeholder, + inheritWidth, ...props } = this.props; return ( @@ -45,7 +64,11 @@ export default (PickerComponent, NameComponent, type) => {({ onClose }) => ( { From c7ec4a6391ad01d6e5fe7a05cfd4d8b2d87fa2f5 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Wed, 11 Jul 2018 13:54:20 -0700 Subject: [PATCH 065/116] Make add question to dashboard modal not full page --- frontend/src/metabase/dashboard/components/DashboardHeader.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/metabase/dashboard/components/DashboardHeader.jsx b/frontend/src/metabase/dashboard/components/DashboardHeader.jsx index 59a2d4712eeef..eab5200cc0241 100644 --- a/frontend/src/metabase/dashboard/components/DashboardHeader.jsx +++ b/frontend/src/metabase/dashboard/components/DashboardHeader.jsx @@ -207,7 +207,6 @@ export default class DashboardHeader extends Component { if (!isFullscreen && canEdit) { buttons.push( Date: Wed, 11 Jul 2018 14:01:30 -0700 Subject: [PATCH 066/116] Hydrating personal_collection_id should force creation of Personal Collections --- src/metabase/models/collection.clj | 7 ++++++- test/metabase/models/collection_test.clj | 13 +++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/metabase/models/collection.clj b/src/metabase/models/collection.clj index d4f883758bcd3..a6d9a4d0f2f2b 100644 --- a/src/metabase/models/collection.clj +++ b/src/metabase/models/collection.clj @@ -1002,7 +1002,12 @@ {:batched-hydrate :personal_collection_id} [users] (when (seq users) + ;; efficiently create a map of user ID -> personal collection ID (let [user-id->collection-id (db/select-field->id :personal_owner_id Collection :personal_owner_id [:in (set (map u/get-id users))])] + ;; now for each User, try to find the corresponding ID out of that map. If it's not present (the personal + ;; Collection hasn't been created yet), then instead call `user->personal-collection-id`, which will create it + ;; as a side-effect. This will ensure this property never comes back as `nil` (for [user users] - (assoc user :personal_collection_id (user-id->collection-id (u/get-id user))))))) + (assoc user :personal_collection_id (or (user-id->collection-id (u/get-id user)) + (user->personal-collection-id (u/get-id user)))))))) diff --git a/test/metabase/models/collection_test.clj b/test/metabase/models/collection_test.clj index 766410e9d8239..159aad11adb1b 100644 --- a/test/metabase/models/collection_test.clj +++ b/test/metabase/models/collection_test.clj @@ -7,14 +7,16 @@ [card :refer [Card]] [collection :as collection :refer [Collection]] [dashboard :refer [Dashboard]] - [permissions :refer [Permissions] :as perms] + [permissions :as perms :refer [Permissions]] [permissions-group :as group :refer [PermissionsGroup]] [pulse :refer [Pulse]] [user :refer [User]]] [metabase.test.data.users :as test-users] [metabase.test.util :as tu] [metabase.util :as u] - [toucan.db :as db] + [toucan + [db :as db] + [hydrate :refer [hydrate]]] [toucan.util.test :as tt])) (defn force-create-personal-collections! @@ -1460,6 +1462,13 @@ (let [personal-collection (collection/user->personal-collection my-cool-user)] (db/update! Collection (u/get-id personal-collection) :personal_owner_id (test-users/user->id :crowberto))))) +;; Does hydrating `:personal_collection_id` force creation of Personal Collections? +(expect + (tt/with-temp User [temp-user] + (-> (hydrate temp-user :personal_collection_id) + :personal_collection_id + integer?))) + ;;; +----------------------------------------------------------------------------------------------------------------+ ;;; | Moving Collections "Across the Boundary" | From e9d41c1c933578143138d41ba7667531217a331a Mon Sep 17 00:00:00 2001 From: Cam Saul Date: Wed, 11 Jul 2018 13:39:26 -0700 Subject: [PATCH 067/116] Tests for MultiMetaBot am-i-the-metabot? functionality --- test/metabase/metabot_test.clj | 37 ++++++++++++++++++++++++ test/metabase/models/collection_test.clj | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 test/metabase/metabot_test.clj diff --git a/test/metabase/metabot_test.clj b/test/metabase/metabot_test.clj new file mode 100644 index 0000000000000..be6bee6467c2e --- /dev/null +++ b/test/metabase/metabot_test.clj @@ -0,0 +1,37 @@ +(ns metabase.metabot-test + (:require [expectations :refer :all] + [metabase.metabot :as metabot] + [metabase.util.date :as du])) + +;; test that if we're not the MetaBot based on Settings, our function to check is working correctly +(expect + false + (do + (#'metabot/metabot-instance-uuid nil) + (#'metabot/metabot-instance-last-checkin nil) + (#'metabot/am-i-the-metabot?))) + +;; test that if nobody is currently the MetaBot, we will become the MetaBot +(expect + (do + (#'metabot/metabot-instance-uuid nil) + (#'metabot/metabot-instance-last-checkin nil) + (#'metabot/check-and-update-instance-status!) + (#'metabot/am-i-the-metabot?))) + +;; test that if nobody has checked in as MetaBot for a while, we will become the MetaBot +(expect + (do + (#'metabot/metabot-instance-uuid (str (java.util.UUID/randomUUID))) + (#'metabot/metabot-instance-last-checkin (du/relative-date :minute -10 (#'metabot/current-timestamp-from-db))) + (#'metabot/check-and-update-instance-status!) + (#'metabot/am-i-the-metabot?))) + +;; check that if another instance has checked in recently, we will *not* become the MetaBot +(expect + false + (do + (#'metabot/metabot-instance-uuid (str (java.util.UUID/randomUUID))) + (#'metabot/metabot-instance-last-checkin (#'metabot/current-timestamp-from-db)) + (#'metabot/check-and-update-instance-status!) + (#'metabot/am-i-the-metabot?))) diff --git a/test/metabase/models/collection_test.clj b/test/metabase/models/collection_test.clj index 766410e9d8239..4da3b4acc6b13 100644 --- a/test/metabase/models/collection_test.clj +++ b/test/metabase/models/collection_test.clj @@ -7,7 +7,7 @@ [card :refer [Card]] [collection :as collection :refer [Collection]] [dashboard :refer [Dashboard]] - [permissions :refer [Permissions] :as perms] + [permissions :as perms :refer [Permissions]] [permissions-group :as group :refer [PermissionsGroup]] [pulse :refer [Pulse]] [user :refer [User]]] From fa8d11a8e26de9e48eaf66ba5ced5c960f1b469e Mon Sep 17 00:00:00 2001 From: Cam Saul Date: Wed, 11 Jul 2018 14:20:53 -0700 Subject: [PATCH 068/116] Code cleanup :shower: --- src/metabase/db.clj | 2 +- src/metabase/util/date.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metabase/db.clj b/src/metabase/db.clj index 2e0c5e3cea538..0d6349c45b560 100644 --- a/src/metabase/db.clj +++ b/src/metabase/db.clj @@ -10,8 +10,8 @@ [metabase [config :as config] [util :as u]] - [puppetlabs.i18n.core :refer [trs]] [metabase.db.spec :as dbspec] + [puppetlabs.i18n.core :refer [trs]] [ring.util.codec :as codec] [toucan.db :as db]) (:import com.mchange.v2.c3p0.ComboPooledDataSource diff --git a/src/metabase/util/date.clj b/src/metabase/util/date.clj index a932f866e96c3..004b9a8e9e704 100644 --- a/src/metabase/util/date.clj +++ b/src/metabase/util/date.clj @@ -113,7 +113,7 @@ on clj-time to ensure correct conversions between the various types NOTE: This function requires you to pass in a timezone or bind `*report-timezone*`, probably to make sure you're not - doing something dumb by forgetting it.For cases where you'd just like to parse an ISO-8601-encoded String in peace + doing something dumb by forgetting it. For cases where you'd just like to parse an ISO-8601-encoded String in peace without specifying a timezone, pass in `:no-timezone` as the second param to explicitly have things parsed without one." ([coercible-to-ts] From af4a4bc9a08318f07581c282c9086c1b0ea94b97 Mon Sep 17 00:00:00 2001 From: Kyle Doherty Date: Wed, 11 Jul 2018 14:29:01 -0700 Subject: [PATCH 069/116] mobile icons and width (#8034) --- .../src/metabase/components/ItemTypeFilterBar.jsx | 11 +++++++++-- frontend/src/metabase/home/containers/SearchApp.jsx | 9 ++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/frontend/src/metabase/components/ItemTypeFilterBar.jsx b/frontend/src/metabase/components/ItemTypeFilterBar.jsx index 9be9685b03b08..5e2cbf2b0b0ba 100644 --- a/frontend/src/metabase/components/ItemTypeFilterBar.jsx +++ b/frontend/src/metabase/components/ItemTypeFilterBar.jsx @@ -3,6 +3,7 @@ import { Flex } from "grid-styled"; import { t } from "c-3po"; import { withRouter } from "react-router"; +import Icon from "metabase/components/Icon"; import Link from "metabase/components/Link"; import colors from "metabase/lib/colors"; @@ -11,18 +12,22 @@ export const FILTERS = [ { name: t`Everything`, filter: null, + icon: "list", }, { name: t`Dashboards`, filter: "dashboard", + icon: "dashboard", }, { name: t`Questions`, filter: "card", + icon: "beaker", }, { name: t`Pulses`, filter: "pulse", + icon: "pulse", }, ]; @@ -47,7 +52,8 @@ const ItemTypeFilterBar = props => { }} color={color} hover={{ color: colors.brand }} - mr={2} + className="flex-full flex align-center justify-center sm-block" + mr={[0, 2]} py={1} style={{ borderBottom: `2px solid ${ @@ -55,8 +61,9 @@ const ItemTypeFilterBar = props => { }`, }} > +
- + + {jt`Results for "${location.query.q}"`} - + Date: Wed, 11 Jul 2018 14:30:12 -0700 Subject: [PATCH 070/116] make entity icons a little bigger --- frontend/src/metabase/components/EntityItem.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/metabase/components/EntityItem.jsx b/frontend/src/metabase/components/EntityItem.jsx index b822a38255709..fef337e38ca07 100644 --- a/frontend/src/metabase/components/EntityItem.jsx +++ b/frontend/src/metabase/components/EntityItem.jsx @@ -80,11 +80,11 @@ const EntityItem = ({ {selectable ? ( } - swappedElement={} + defaultElement={} + swappedElement={} /> ) : ( - + )}

From ef937ab2017022c0deaf0a1bccb9d322986e93d5 Mon Sep 17 00:00:00 2001 From: Kyle Doherty Date: Wed, 11 Jul 2018 14:39:21 -0700 Subject: [PATCH 071/116] change fixed height --- frontend/src/metabase/components/CollectionLanding.jsx | 4 ++-- frontend/src/metabase/components/EntityItem.jsx | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/frontend/src/metabase/components/CollectionLanding.jsx b/frontend/src/metabase/components/CollectionLanding.jsx index 6f69ddc832de0..1d27362963e93 100644 --- a/frontend/src/metabase/components/CollectionLanding.jsx +++ b/frontend/src/metabase/components/CollectionLanding.jsx @@ -38,7 +38,7 @@ import PinPositionDropTarget from "metabase/containers/dnd/PinPositionDropTarget import PinDropTarget from "metabase/containers/dnd/PinDropTarget"; import ItemsDragLayer from "metabase/containers/dnd/ItemsDragLayer"; -const ROW_HEIGHT = 72; +const ROW_HEIGHT = 86; import { entityListLoader } from "metabase/entities/containers/EntityListLoader"; @@ -308,7 +308,7 @@ export const NormalItem = ({ onToggleSelected, onMove, }) => ( - + 0} selectable diff --git a/frontend/src/metabase/components/EntityItem.jsx b/frontend/src/metabase/components/EntityItem.jsx index e9000706e57fc..5180a875bc8fb 100644 --- a/frontend/src/metabase/components/EntityItem.jsx +++ b/frontend/src/metabase/components/EntityItem.jsx @@ -21,9 +21,7 @@ const EntityItemWrapper = Flex.extend` `; export const EntityListItem = props => ( - - - + ); export const EntityCardItem = props => ( From 2ffd434a1005e204692bbde48add1521a0d47e35 Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Wed, 11 Jul 2018 15:09:11 -0700 Subject: [PATCH 072/116] dial in lists --- frontend/src/metabase/components/Card.jsx | 1 + .../src/metabase/components/CollectionLanding.jsx | 2 +- frontend/src/metabase/components/EntityItem.jsx | 14 +++++++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/frontend/src/metabase/components/Card.jsx b/frontend/src/metabase/components/Card.jsx index faf07b1cffbd3..13b9acedd4ba5 100644 --- a/frontend/src/metabase/components/Card.jsx +++ b/frontend/src/metabase/components/Card.jsx @@ -11,6 +11,7 @@ const Card = styled.div` border-radius: 6px; box-shadow: 0 7px 20px ${props => colors["shadow"]}; transition: all 0.2s linear; + line-height: 24px; ${props => props.hoverable && `&:hover { diff --git a/frontend/src/metabase/components/CollectionLanding.jsx b/frontend/src/metabase/components/CollectionLanding.jsx index 1d27362963e93..73b35294b1065 100644 --- a/frontend/src/metabase/components/CollectionLanding.jsx +++ b/frontend/src/metabase/components/CollectionLanding.jsx @@ -38,7 +38,7 @@ import PinPositionDropTarget from "metabase/containers/dnd/PinPositionDropTarget import PinDropTarget from "metabase/containers/dnd/PinDropTarget"; import ItemsDragLayer from "metabase/containers/dnd/ItemsDragLayer"; -const ROW_HEIGHT = 86; +const ROW_HEIGHT = 72; import { entityListLoader } from "metabase/entities/containers/EntityListLoader"; diff --git a/frontend/src/metabase/components/EntityItem.jsx b/frontend/src/metabase/components/EntityItem.jsx index 5180a875bc8fb..a711682fd47e2 100644 --- a/frontend/src/metabase/components/EntityItem.jsx +++ b/frontend/src/metabase/components/EntityItem.jsx @@ -20,9 +20,7 @@ const EntityItemWrapper = Flex.extend` } `; -export const EntityListItem = props => ( - -); +export const EntityListItem = props => ; export const EntityCardItem = props => ( @@ -64,7 +62,7 @@ const EntityItem = ({ return ( } - swappedElement={} + defaultElement={ + + } + swappedElement={} /> ) : ( - + )}

From 001381774216792767cc23a30fb1fbfcb352c4ce Mon Sep 17 00:00:00 2001 From: Maz Ameli Date: Wed, 11 Jul 2018 15:12:04 -0700 Subject: [PATCH 073/116] update snapshots --- .../internal/__snapshots__/components.unit.spec.js.snap | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/test/internal/__snapshots__/components.unit.spec.js.snap b/frontend/test/internal/__snapshots__/components.unit.spec.js.snap index 70fb8030fb5b2..cc07a5ea1568a 100644 --- a/frontend/test/internal/__snapshots__/components.unit.spec.js.snap +++ b/frontend/test/internal/__snapshots__/components.unit.spec.js.snap @@ -2,7 +2,7 @@ exports[`Card should render "dark" correctly 1`] = `
Date: Wed, 11 Jul 2018 18:23:36 -0700 Subject: [PATCH 074/116] Responsive collections (#8035) * collection sizing on mobile * fix item width * prettier --- .../metabase/components/CollectionLanding.jsx | 89 +++++++++---------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/frontend/src/metabase/components/CollectionLanding.jsx b/frontend/src/metabase/components/CollectionLanding.jsx index 73b35294b1065..77ee2b09c8132 100644 --- a/frontend/src/metabase/components/CollectionLanding.jsx +++ b/frontend/src/metabase/components/CollectionLanding.jsx @@ -15,7 +15,7 @@ import Button from "metabase/components/Button"; import Card from "metabase/components/Card"; import Modal from "metabase/components/Modal"; import StackedCheckBox from "metabase/components/StackedCheckBox"; -import EntityListItem from "metabase/components/EntityItem"; +import EntityItem from "metabase/components/EntityItem"; import { Grid, GridItem } from "metabase/components/Grid"; import Icon from "metabase/components/Icon"; import Link from "metabase/components/Link"; @@ -39,6 +39,7 @@ import PinDropTarget from "metabase/containers/dnd/PinDropTarget"; import ItemsDragLayer from "metabase/containers/dnd/ItemsDragLayer"; const ROW_HEIGHT = 72; +const PAGE_PADDING = [2, 3, 4]; import { entityListLoader } from "metabase/entities/containers/EntityListLoader"; @@ -97,9 +98,9 @@ class DefaultLanding extends React.Component { onSelectNone(); }; - const collectionWidth = unpinned.length > 0 ? 1 / 3 : 1; - const itemWidth = unpinned.length > 0 ? 2 / 3 : 0; - const collectionGridSize = unpinned.length > 0 ? 1 : 1 / 4; + const collectionWidth = unpinned.length > 0 ? [1, 1 / 3] : 1; + const itemWidth = unpinned.length > 0 ? [1, 2 / 3] : 0; + const collectionGridSize = unpinned.length > 0 ? 1 : [1, 1 / 4]; return ( @@ -107,7 +108,7 @@ class DefaultLanding extends React.Component { {pinned.length > 0 ? ( - + {t`Pins`} {pinned.map((item, index) => ( - + )} - + - + {t`Collections`} @@ -308,8 +309,8 @@ export const NormalItem = ({ onToggleSelected, onMove, }) => ( - - + 0} selectable item={item} @@ -413,45 +414,41 @@ class CollectionLanding extends React.Component { return ( - - - - - ({ - title: ( - - {name} - - ), - to: Urls.collection(id), - })), - ]} - /> - -

- {currentCollection.name} -

+ + + + ({ + title: ( + + {name} + + ), + to: Urls.collection(id), + })), + ]} + /> +

{currentCollection.name}

+ - - {currentCollection && - currentCollection.can_write && - !currentCollection.personal_owner_id && ( - - - - )} - - - - + + {currentCollection && + currentCollection.can_write && + !currentCollection.personal_owner_id && ( + + + + )} + + + - + Date: Wed, 11 Jul 2018 20:12:54 -0700 Subject: [PATCH 075/116] responsive data browser (#8041) --- frontend/src/metabase/components/BrowseApp.jsx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/frontend/src/metabase/components/BrowseApp.jsx b/frontend/src/metabase/components/BrowseApp.jsx index 7caabfc49d015..23bc6034a1a07 100644 --- a/frontend/src/metabase/components/BrowseApp.jsx +++ b/frontend/src/metabase/components/BrowseApp.jsx @@ -21,6 +21,9 @@ export const DatabaseListLoader = props => ( ); +const PAGE_PADDING = [1, 2, 4]; +const ITEM_WIDTHS = [1, 1 / 2, 1 / 3]; + const SchemaListLoader = ({ dbId, ...props }) => ( ); @@ -63,7 +66,7 @@ export class SchemaBrowser extends React.Component { {schemas.map(schema => ( - + + {this.props.children}; + return {this.props.children}; } } @@ -199,7 +202,7 @@ export class DatabaseBrowser extends React.Component { return ( {databases.map(database => ( - + From 1dc881f7748a4dd6302227ee9e9c9ca384e89f8c Mon Sep 17 00:00:00 2001 From: Kyle Doherty Date: Thu, 12 Jul 2018 09:47:07 -0700 Subject: [PATCH 076/116] icon size --- frontend/src/metabase/containers/ItemPicker.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/metabase/containers/ItemPicker.jsx b/frontend/src/metabase/containers/ItemPicker.jsx index 22ec5b9890464..63c37bb1dbca8 100644 --- a/frontend/src/metabase/containers/ItemPicker.jsx +++ b/frontend/src/metabase/containers/ItemPicker.jsx @@ -223,7 +223,7 @@ const Item = ({ })} > - +

{name}

{hasChildren && ( Date: Thu, 12 Jul 2018 14:51:32 -0500 Subject: [PATCH 077/116] Bump Liquibase to 3.6.2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 76ed1ee742832..5e154b075b8ea 100644 --- a/project.clj +++ b/project.clj @@ -83,7 +83,7 @@ [net.sf.cssbox/cssbox "4.12" ; HTML / CSS rendering :exclusions [org.slf4j/slf4j-api]] [org.clojars.pntblnk/clj-ldap "0.0.12"] ; LDAP client - [org.liquibase/liquibase-core "3.5.3"] ; migration management (Java lib) + [org.liquibase/liquibase-core "3.6.2"] ; migration management (Java lib) [org.postgresql/postgresql "42.2.2"] ; Postgres driver [org.slf4j/slf4j-log4j12 "1.7.25"] ; abstraction for logging frameworks -- allows end user to plug in desired logging framework at deployment time [org.tcrawley/dynapath "0.2.5"] ; Dynamically add Jars (e.g. Oracle or Vertica) to classpath From 8deb068a83ca9b6104c1e7c44cf58caedf42f8fd Mon Sep 17 00:00:00 2001 From: Kyle Doherty Date: Thu, 12 Jul 2018 13:27:35 -0700 Subject: [PATCH 078/116] Component and color tweaks (#8038) * fix color names * separate and document collection item * document entity item * fix keys * clean up layout props * fix warnings * prettier * block bordered prop * key * section keys * collection color changes * fix padding * list variant * update snap * check for location * rm documented components for now --- .../admin/people/components/GroupDetail.jsx | 2 +- .../src/metabase/components/ArchivedItem.jsx | 2 +- .../metabase/components/CollectionItem.jsx | 30 +++ .../metabase/components/CollectionLanding.jsx | 179 ++++++++++-------- .../metabase/components/CollectionList.jsx | 23 +-- .../src/metabase/components/EntityItem.jsx | 35 +++- .../src/metabase/components/ErrorDetails.jsx | 2 +- .../src/metabase/components/ExplorePane.jsx | 2 +- frontend/src/metabase/components/Grid.jsx | 2 +- frontend/src/metabase/components/Icon.jsx | 3 +- .../src/metabase/components/IconWrapper.jsx | 2 +- .../metabase/components/ItemTypeFilterBar.jsx | 3 +- frontend/src/metabase/components/Link.jsx | 7 +- .../src/metabase/components/ToggleLarge.jsx | 2 +- .../src/metabase/components/TokenField.jsx | 6 +- .../src/metabase/containers/EntitySearch.jsx | 16 +- .../src/metabase/containers/Overworld.jsx | 7 +- .../src/metabase/containers/dnd/DropArea.jsx | 2 +- frontend/src/metabase/css/core/colors.css | 48 ++--- frontend/src/metabase/css/pulse.css | 4 - .../containers/AutomaticDashboardApp.jsx | 4 +- .../src/metabase/home/components/Activity.jsx | 2 +- .../metabase/home/containers/SearchApp.jsx | 8 +- .../internal/components/ComponentsApp.jsx | 9 +- frontend/src/metabase/lib/formatting.js | 2 +- frontend/src/metabase/lib/utils.js | 33 ++++ .../widgets/AdvancedSettingsPane.jsx | 5 +- .../pulse/components/PulseEditChannels.jsx | 2 +- .../pulse/components/PulseListItem.jsx | 2 +- .../components/AlertListPopoverContent.jsx | 2 +- .../query_builder/components/AlertModals.jsx | 4 +- .../query_builder/components/DataSelector.jsx | 4 +- .../settings/ChartSettingsTableFormatting.jsx | 2 +- .../visualizations/visualizations/Text.jsx | 2 +- .../components.unit.spec.js.snap | 16 +- 35 files changed, 263 insertions(+), 211 deletions(-) create mode 100644 frontend/src/metabase/components/CollectionItem.jsx diff --git a/frontend/src/metabase/admin/people/components/GroupDetail.jsx b/frontend/src/metabase/admin/people/components/GroupDetail.jsx index 2d8d5d647edc1..add1f3a65aec1 100644 --- a/frontend/src/metabase/admin/people/components/GroupDetail.jsx +++ b/frontend/src/metabase/admin/people/components/GroupDetail.jsx @@ -115,7 +115,7 @@ const AddUserRow = ({ onCancel={onCancel} > {selectedUsers.map(user => ( -
+
{user.common_name} ( -
+
( + + + +

+ {collection.name} +

+ + +); + +export default CollectionItem; diff --git a/frontend/src/metabase/components/CollectionLanding.jsx b/frontend/src/metabase/components/CollectionLanding.jsx index 77ee2b09c8132..746f414d41e3e 100644 --- a/frontend/src/metabase/components/CollectionLanding.jsx +++ b/frontend/src/metabase/components/CollectionLanding.jsx @@ -75,6 +75,7 @@ class DefaultLanding extends React.Component { render() { const { + ancestors, collection, collectionId, @@ -82,6 +83,7 @@ class DefaultLanding extends React.Component { pinned, unpinned, + isRoot, selected, selection, onToggleSelected, @@ -105,10 +107,51 @@ class DefaultLanding extends React.Component { return ( + + + + ({ + title: ( + + {name} + + ), + to: Urls.collection(id), + })), + ]} + /> + +

{collection.name}

+ + + + {collection && + collection.can_write && + !collection.personal_owner_id && ( + + + + )} + + + + + {pinned.length > 0 ? ( - + {t`Pins`} {pinned.map((item, index) => ( - + )} - + @@ -186,41 +233,43 @@ class DefaultLanding extends React.Component { - 0 ? 5 : 2} - style={{ - position: "relative", - height: ROW_HEIGHT * unpinned.length, - }} - > - u.model === location.query.type, - ) - : unpinned - } - rowHeight={ROW_HEIGHT} - renderItem={({ item, index }) => ( - - + 0 ? 5 : 2} + style={{ + position: "relative", + height: ROW_HEIGHT * unpinned.length, + }} + > + u.model === location.query.type, + ) + : unpinned + } + rowHeight={ROW_HEIGHT} + renderItem={({ item, index }) => ( + - this.setState({ moveItems }) - } - /> - - )} - /> - + > + + this.setState({ moveItems }) + } + /> + + )} + /> + + ) : ( @@ -309,8 +358,9 @@ export const NormalItem = ({ onToggleSelected, onMove, }) => ( - + 0} selectable item={item} @@ -414,51 +464,16 @@ class CollectionLanding extends React.Component { return ( - - - - ({ - title: ( - - {name} - - ), - to: Urls.collection(id), - })), - ]} - /> - -

{currentCollection.name}

- - - - {currentCollection && - currentCollection.can_write && - !currentCollection.personal_owner_id && ( - - - - )} - - - - - - - - { - // Need to have this here so the child modals will show up - this.props.children - } - + + { + // Need to have this here so the child modals will show up + this.props.children + } ); } diff --git a/frontend/src/metabase/components/CollectionList.jsx b/frontend/src/metabase/components/CollectionList.jsx index 48f3a54201857..2bc0bc3a15354 100644 --- a/frontend/src/metabase/components/CollectionList.jsx +++ b/frontend/src/metabase/components/CollectionList.jsx @@ -3,10 +3,10 @@ import { t } from "c-3po"; import { Box, Flex } from "grid-styled"; import { connect } from "react-redux"; -import colors, { normal } from "metabase/lib/colors"; import * as Urls from "metabase/lib/urls"; -import Ellipsified from "metabase/components/Ellipsified"; +import CollectionItem from "metabase/components/CollectionItem"; +import { normal } from "metabase/lib/colors"; import { Grid, GridItem } from "metabase/components/Grid"; import Icon from "metabase/components/Icon"; import Link from "metabase/components/Link"; @@ -14,23 +14,6 @@ import Link from "metabase/components/Link"; import CollectionDropTarget from "metabase/containers/dnd/CollectionDropTarget"; import ItemDragSource from "metabase/containers/dnd/ItemDragSource"; -const CollectionItem = ({ collection, color, iconName = "all" }) => ( - - - - -

- {collection.name} -

- - - -); - @connect(({ currentUser }) => ({ currentUser }), null) class CollectionList extends React.Component { render() { @@ -47,7 +30,7 @@ class CollectionList extends React.Component { {collections .filter(c => c.id !== currentUser.personal_collection_id) .map(collection => ( - + diff --git a/frontend/src/metabase/components/EntityItem.jsx b/frontend/src/metabase/components/EntityItem.jsx index a711682fd47e2..e341992e3f659 100644 --- a/frontend/src/metabase/components/EntityItem.jsx +++ b/frontend/src/metabase/components/EntityItem.jsx @@ -1,7 +1,8 @@ import React from "react"; import { t } from "c-3po"; +import cx from "classnames"; +import { Flex } from "grid-styled"; -import { Box, Flex } from "grid-styled"; import EntityMenu from "metabase/components/EntityMenu"; import Swapper from "metabase/components/Swapper"; import IconWrapper from "metabase/components/IconWrapper"; @@ -20,14 +21,6 @@ const EntityItemWrapper = Flex.extend` } `; -export const EntityListItem = props => ; - -export const EntityCardItem = props => ( - - - -); - const EntityItem = ({ name, iconName, @@ -40,6 +33,7 @@ const EntityItem = ({ selected, onToggleSelected, selectable, + variant, }) => { const actions = [ onPin && { @@ -59,8 +53,29 @@ const EntityItem = ({ }, ].filter(action => action); + let spacing; + + switch (variant) { + case "list": + spacing = { + px: 2, + py: 2, + }; + break; + default: + spacing = { + py: 2, + }; + break; + } + return ( - + {t`Here's the full error message`}

{/* ensure we don't try to render anything except a string */} {typeof details === "string" diff --git a/frontend/src/metabase/components/ExplorePane.jsx b/frontend/src/metabase/components/ExplorePane.jsx index f5f062d293485..4ee8aee50e161 100644 --- a/frontend/src/metabase/components/ExplorePane.jsx +++ b/frontend/src/metabase/components/ExplorePane.jsx @@ -158,7 +158,7 @@ export const ExploreList = ({ export const ExploreOption = ({ option }: { option: Candidate }) => ( ( - + {children} ); diff --git a/frontend/src/metabase/components/Icon.jsx b/frontend/src/metabase/components/Icon.jsx index cc1d98c3be865..5d3221a19e41a 100644 --- a/frontend/src/metabase/components/Icon.jsx +++ b/frontend/src/metabase/components/Icon.jsx @@ -7,6 +7,7 @@ import { color, space, hover } from "styled-system"; import cx from "classnames"; import { loadIcon } from "metabase/icon_paths"; +import { stripLayoutProps } from "metabase/lib/utils"; import Tooltipify from "metabase/hoc/Tooltipify"; @@ -56,7 +57,7 @@ class BaseIcon extends Component { return ; } else { return ( - + ); diff --git a/frontend/src/metabase/components/IconWrapper.jsx b/frontend/src/metabase/components/IconWrapper.jsx index 27120f2fbf966..5deb01190c325 100644 --- a/frontend/src/metabase/components/IconWrapper.jsx +++ b/frontend/src/metabase/components/IconWrapper.jsx @@ -2,7 +2,7 @@ import { Flex } from "grid-styled"; import colors from "metabase/lib/colors"; const IconWrapper = Flex.extend` - background: ${props => colors["bg-light"]}; + background: ${colors["bg-medium"]}; border-radius: 6px; `; diff --git a/frontend/src/metabase/components/ItemTypeFilterBar.jsx b/frontend/src/metabase/components/ItemTypeFilterBar.jsx index 72d95c6fac0b0..4953391d024b6 100644 --- a/frontend/src/metabase/components/ItemTypeFilterBar.jsx +++ b/frontend/src/metabase/components/ItemTypeFilterBar.jsx @@ -36,7 +36,7 @@ const ItemTypeFilterBar = props => { return ( {props.filters.map(f => { - let isActive = location.query.type === f.filter; + let isActive = location && location.query.type === f.filter; if (!location.query.type && !f.filter) { isActive = true; @@ -54,6 +54,7 @@ const ItemTypeFilterBar = props => { hover={{ color: colors.brand }} className="flex-full flex align-center justify-center sm-block" mr={[0, 2]} + key={f.filter} py={1} style={{ borderBottom: `2px solid ${ diff --git a/frontend/src/metabase/components/Link.jsx b/frontend/src/metabase/components/Link.jsx index 34cf81f5c560d..15cafa2f180d8 100644 --- a/frontend/src/metabase/components/Link.jsx +++ b/frontend/src/metabase/components/Link.jsx @@ -2,9 +2,14 @@ import React from "react"; import { Link as ReactRouterLink } from "react-router"; import styled from "styled-components"; import { display, color, hover, space } from "styled-system"; +import { stripLayoutProps } from "metabase/lib/utils"; const BaseLink = ({ to, className, children, ...props }) => ( - + {children} ); diff --git a/frontend/src/metabase/components/ToggleLarge.jsx b/frontend/src/metabase/components/ToggleLarge.jsx index 964b9d624feb6..96307e6cac878 100644 --- a/frontend/src/metabase/components/ToggleLarge.jsx +++ b/frontend/src/metabase/components/ToggleLarge.jsx @@ -11,7 +11,7 @@ const ToggleLarge = ({ textRight, }) => (
@@ -612,9 +612,9 @@ export default class TokenField extends Component { } className={cx( `py1 pl1 pr2 block rounded text-bold text-${color}-hover inline-block full cursor-pointer`, - `bg-grey-0-hover`, + `bg-light-hover`, { - [`text-${color} bg-grey-0`]: + [`text-${color} bg-light`]: !this.state.listIsHovered && this._valueIsEqual( selectedOptionValue, diff --git a/frontend/src/metabase/containers/EntitySearch.jsx b/frontend/src/metabase/containers/EntitySearch.jsx index 781b783faa662..89af5acc90257 100644 --- a/frontend/src/metabase/containers/EntitySearch.jsx +++ b/frontend/src/metabase/containers/EntitySearch.jsx @@ -185,7 +185,7 @@ export default class EntitySearch extends Component { filteredEntities.length > 0; return ( -
+
(
{groupName !== null && ( -
+
@@ -510,8 +510,8 @@ class SearchResultsList extends Component { !isInEnd && setCurrentPage(entities, currentPage + 1)} @@ -587,8 +587,8 @@ export class SearchResultListItem extends Component {
  • diff --git a/frontend/src/metabase/containers/Overworld.jsx b/frontend/src/metabase/containers/Overworld.jsx index 32bc06c1c33de..61eb23bf74974 100644 --- a/frontend/src/metabase/containers/Overworld.jsx +++ b/frontend/src/metabase/containers/Overworld.jsx @@ -109,7 +109,10 @@ class Overworld extends React.Component { {pinnedDashboards.map(pin => { return ( - + {databases.map(database => ( - + ( } >
    @@ -246,7 +246,7 @@ const SuggestionsList = ({ suggestions, section }) => ( ); const SuggestionsSidebar = ({ related }) => ( -
    +

    More X-rays

    diff --git a/frontend/src/metabase/home/components/Activity.jsx b/frontend/src/metabase/home/components/Activity.jsx index c4fc62ccd71c2..7a31be22737c3 100644 --- a/frontend/src/metabase/home/components/Activity.jsx +++ b/frontend/src/metabase/home/components/Activity.jsx @@ -21,7 +21,7 @@ export default class Activity extends Component { "bg-error", "bg-green", "bg-gold", - "bg-grey-2", + "bg-medium", ]; } diff --git a/frontend/src/metabase/home/containers/SearchApp.jsx b/frontend/src/metabase/home/containers/SearchApp.jsx index 3c35ad0660726..e02312b06468a 100644 --- a/frontend/src/metabase/home/containers/SearchApp.jsx +++ b/frontend/src/metabase/home/containers/SearchApp.jsx @@ -70,7 +70,7 @@ export default class SearchApp extends React.Component {
    {types.dashboard.map(item => ( - + {types.collection.map(item => ( - + {types.card.map(item => ( - + {types.pulse.map(item => ( - + -
    +
    {COMPONENTS.filter( ({ component, description, examples }) => !componentName || componentName === slugify(component.name), - ).map(({ component, description, examples }) => ( -
    + ).map(({ component, description, examples }, index) => ( +

    { + if (Object.keys(newProps).includes(l)) { + delete newProps[l]; + } + }); + + return newProps; +} + function s4() { return Math.floor((1 + Math.random()) * 0x10000) .toString(16) diff --git a/frontend/src/metabase/public/components/widgets/AdvancedSettingsPane.jsx b/frontend/src/metabase/public/components/widgets/AdvancedSettingsPane.jsx index 783ff8bcf8f81..64ad10bfe22ee 100644 --- a/frontend/src/metabase/public/components/widgets/AdvancedSettingsPane.jsx +++ b/frontend/src/metabase/public/components/widgets/AdvancedSettingsPane.jsx @@ -64,10 +64,7 @@ const AdvancedSettingsPane = ({ onChangeParameterValue, }: Props) => (
    diff --git a/frontend/src/metabase/pulse/components/PulseEditChannels.jsx b/frontend/src/metabase/pulse/components/PulseEditChannels.jsx index 6dd43b1d1a6e1..cfb60fd8475ef 100644 --- a/frontend/src/metabase/pulse/components/PulseEditChannels.jsx +++ b/frontend/src/metabase/pulse/components/PulseEditChannels.jsx @@ -276,7 +276,7 @@ export default class PulseEditChannels extends Component { />
    {channels.length > 0 && channelSpec.configured ? ( -
      {channels}
    +
      {channels}
    ) : channels.length > 0 && !channelSpec.configured ? (

    {t`${ diff --git a/frontend/src/metabase/pulse/components/PulseListItem.jsx b/frontend/src/metabase/pulse/components/PulseListItem.jsx index db66a0ca628d6..245f15b3ae54c 100644 --- a/frontend/src/metabase/pulse/components/PulseListItem.jsx +++ b/frontend/src/metabase/pulse/components/PulseListItem.jsx @@ -68,7 +68,7 @@ export default class PulseListItem extends Component {

    • ))} -