Skip to content

Commit

Permalink
[feature](backup) backup and restore privileges catalogs workloadgroup (
Browse files Browse the repository at this point in the history 
#44905)
  • Loading branch information
@justfortaste
justfortaste authored Dec 25, 2024
1 parent 041db85 commit 8e57386
Show file tree
Hide file tree
Showing 33 changed files with 1,944 additions and 43 deletions.
12 changes: 6 additions & 6 deletions fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,10 +305,10 @@ unsupportedOtherStatement
| WARM UP (CLUSTER | COMPUTE GROUP) destination=identifier WITH
((CLUSTER | COMPUTE GROUP) source=identifier |
(warmUpItem (AND warmUpItem)*)) FORCE? #warmUpCluster
| BACKUP SNAPSHOT label=multipartIdentifier TO repo=identifier
| BACKUP GLOBAL? SNAPSHOT label=multipartIdentifier TO repo=identifier
((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)* RIGHT_PAREN)?
properties=propertyClause? #backup
| RESTORE SNAPSHOT label=multipartIdentifier FROM repo=identifier
| RESTORE GLOBAL? SNAPSHOT label=multipartIdentifier FROM repo=identifier
((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)* RIGHT_PAREN)?
properties=propertyClause? #restore
| START TRANSACTION (WITH CONSISTENT SNAPSHOT)? #unsupportedStartTransaction
Expand Down Expand Up @@ -359,8 +359,8 @@ unsupportedShowStatement
wildWhere? sortClause? limitClause? #showTabletsFromTable
| SHOW PROPERTY (FOR user=identifierOrText)? wildWhere? #showUserProperties
| SHOW ALL PROPERTIES wildWhere? #showAllProperties
| SHOW BACKUP ((FROM | IN) database=multipartIdentifier)? wildWhere? #showBackup
| SHOW BRIEF? RESTORE ((FROM | IN) database=multipartIdentifier)? wildWhere? #showRestore
| SHOW GLOBAL? BACKUP ((FROM | IN) database=multipartIdentifier)? wildWhere? #showBackup
| SHOW GLOBAL? BRIEF? RESTORE ((FROM | IN) database=multipartIdentifier)? wildWhere? #showRestore
| SHOW RESOURCES wildWhere? sortClause? limitClause? #showResources
| SHOW WORKLOAD GROUPS wildWhere? #showWorkloadGroups
| SHOW SNAPSHOT ON repo=identifier wildWhere? #showSnapshot
Expand Down Expand Up @@ -496,8 +496,8 @@ unsupportedCancelStatement
(COMMA jobIds+=INTEGER_VALUE)* RIGHT_PAREN)? #cancelBuildIndex
| CANCEL DECOMMISSION BACKEND hostPorts+=STRING_LITERAL
(COMMA hostPorts+=STRING_LITERAL)* #cancelDecommisionBackend
| CANCEL BACKUP ((FROM | IN) database=identifier)? #cancelBackup
| CANCEL RESTORE ((FROM | IN) database=identifier)? #cancelRestore
| CANCEL GLOBAL? BACKUP ((FROM | IN) database=identifier)? #cancelBackup
| CANCEL GLOBAL? RESTORE ((FROM | IN) database=identifier)? #cancelRestore
;

supportedAdminStatement
Expand Down
28 changes: 28 additions & 0 deletions fe/fe-core/src/main/cup/sql_parser.cup
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4550,10 +4550,18 @@ show_param ::=
{:
RESULT = new ShowBackupStmt(db, parser.where);
:}
| KW_GLOBAL KW_BACKUP opt_wild_where
{:
RESULT = new ShowBackupStmt(parser.where);
:}
| KW_RESTORE opt_db:db opt_wild_where
{:
RESULT = new ShowRestoreStmt(db, parser.where);
:}
| KW_GLOBAL KW_RESTORE opt_wild_where
{:
RESULT = new ShowRestoreStmt(parser.where);
:}
| KW_BRIEF KW_RESTORE opt_db:db opt_wild_where
{:
RESULT = new ShowRestoreStmt(db, parser.where, true);
Expand Down Expand Up @@ -5069,10 +5077,18 @@ cancel_param ::=
{:
RESULT = new CancelBackupStmt(db, false);
:}
| KW_GLOBAL KW_BACKUP
{:
RESULT = new CancelBackupStmt(false);
:}
| KW_RESTORE opt_db:db
{:
RESULT = new CancelBackupStmt(db, true);
:}
| KW_GLOBAL KW_RESTORE
{:
RESULT = new CancelBackupStmt(true);
:}
| KW_WARM KW_UP KW_JOB opt_wild_where
{:
RESULT = new CancelCloudWarmUpStmt(parser.where);
Expand Down Expand Up @@ -5456,6 +5472,12 @@ backup_stmt ::=
{:
RESULT = new BackupStmt(label, repoName, tblRefClause, properties);
:}
| KW_BACKUP KW_GLOBAL KW_SNAPSHOT job_label:label
KW_TO ident:repoName
opt_properties:properties
{:
RESULT = new BackupStmt(label, repoName, properties);
:}
;

unlock_tables_stmt ::=
Expand Down Expand Up @@ -5557,6 +5579,12 @@ restore_stmt ::=
{:
RESULT = new RestoreStmt(label, repoName, tblRefClause, properties);
:}
| KW_RESTORE KW_GLOBAL KW_SNAPSHOT job_label:label
KW_FROM ident:repoName
opt_properties:properties
{:
RESULT = new RestoreStmt(label, repoName, properties);
:}
;

// Kill statement
Expand Down
16 changes: 13 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/AbstractBackupStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
Expand All @@ -44,16 +45,21 @@ public class AbstractBackupStmt extends DdlStmt {
protected String repoName;
protected AbstractBackupTableRefClause abstractBackupTableRefClause;
protected Map<String, String> properties;

protected long timeoutMs;
public boolean backupGlobal = false;

public AbstractBackupStmt(LabelName labelName, String repoName,
AbstractBackupTableRefClause abstractBackupTableRefClause,
Map<String, String> properties) {
this.labelName = labelName;
Map<String, String> properties, boolean backupGlobal) {
if (backupGlobal) {
this.labelName = new LabelName(FeConstants.INTERNAL_DB_NAME, labelName.getLabelName());
} else {
this.labelName = labelName;
}
this.repoName = repoName;
this.abstractBackupTableRefClause = abstractBackupTableRefClause;
this.properties = properties == null ? Maps.newHashMap() : properties;
this.backupGlobal = backupGlobal;
}

@Override
Expand Down Expand Up @@ -120,6 +126,10 @@ public String getDbName() {
return labelName.getDbName();
}

public boolean isBackupGlobal() {
return backupGlobal;
}

public String getLabel() {
return labelName.getLabelName();
}
Expand Down
80 changes: 79 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/BackupStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
public class BackupStmt extends AbstractBackupStmt implements NotFallbackInParser {
private static final String PROP_TYPE = "type";
public static final String PROP_CONTENT = "content";
public static final String PROP_BACKUP_PRIV = "backup_privilege";
public static final String PROP_BACKUP_CATALOG = "backup_catalog";
public static final String PROP_BACKUP_WORKLOAD_GROUP = "backup_workload_group";

public enum BackupType {
INCREMENTAL, FULL
Expand All @@ -41,11 +44,18 @@ public enum BackupContent {

private BackupType type = BackupType.FULL;
private BackupContent content = BackupContent.ALL;
private boolean backupPriv = false;
private boolean backupCatalog = false;
private boolean backupWorkloadGroup = false;


public BackupStmt(LabelName labelName, String repoName, AbstractBackupTableRefClause abstractBackupTableRefClause,
Map<String, String> properties) {
super(labelName, repoName, abstractBackupTableRefClause, properties);
super(labelName, repoName, abstractBackupTableRefClause, properties, false);
}

public BackupStmt(LabelName labelName, String repoName, Map<String, String> properties) {
super(labelName, repoName, null, properties, true);
}

public long getTimeoutMs() {
Expand All @@ -60,6 +70,18 @@ public BackupContent getContent() {
return content;
}

public boolean isBackupPriv() {
return backupPriv;
}

public boolean isBackupCatalog() {
return backupCatalog;
}

public boolean isBackupWorkloadGroup() {
return backupWorkloadGroup;
}

@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
Expand Down Expand Up @@ -103,6 +125,62 @@ protected void analyzeProperties() throws AnalysisException {
copiedProperties.remove(PROP_CONTENT);
}

// backup_priv
String backupPrivProp = copiedProperties.get(PROP_BACKUP_PRIV);
if (backupPrivProp != null) {
if (backupPrivProp.equalsIgnoreCase("true")) {
backupPriv = true;
} else if (backupPrivProp.equalsIgnoreCase("false")) {
backupPriv = false;
} else {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Invalid backup privileges:" + backupPrivProp);
}
copiedProperties.remove(PROP_BACKUP_PRIV);
}

// backup_catalog
String backupCatalogProp = copiedProperties.get(PROP_BACKUP_CATALOG);
if (backupCatalogProp != null) {
if (backupCatalogProp.equalsIgnoreCase("true")) {
backupCatalog = true;
} else if (backupCatalogProp.equalsIgnoreCase("false")) {
backupCatalog = false;
} else {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Invalid backup catalog:" + backupCatalogProp);
}
copiedProperties.remove(PROP_BACKUP_CATALOG);
}

// backup_workload
String backupWorkloadGroupProp = copiedProperties.get(PROP_BACKUP_WORKLOAD_GROUP);
if (backupWorkloadGroupProp != null) {
if (backupWorkloadGroupProp.equalsIgnoreCase("true")) {
backupWorkloadGroup = true;
} else if (backupWorkloadGroupProp.equalsIgnoreCase("false")) {
backupWorkloadGroup = false;
} else {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Invalid backup workload group:" + backupWorkloadGroupProp);
}
copiedProperties.remove(PROP_BACKUP_WORKLOAD_GROUP);
}

if (isBackupGlobal()) {
if (properties.get(PROP_BACKUP_PRIV) == null
&& properties.get(PROP_BACKUP_CATALOG) == null
&& properties.get(PROP_BACKUP_WORKLOAD_GROUP) == null) {
backupPriv = true;
backupCatalog = true;
backupWorkloadGroup = true;
}
} else {
backupPriv = false;
backupCatalog = false;
backupWorkloadGroup = false;
}

if (!copiedProperties.isEmpty()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Unknown backup job properties: " + copiedProperties.keySet());
Expand Down
6 changes: 6 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/CancelBackupStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
Expand All @@ -38,6 +39,11 @@ public CancelBackupStmt(String dbName, boolean isRestore) {
this.isRestore = isRestore;
}

public CancelBackupStmt(boolean isRestore) {
this.dbName = FeConstants.INTERNAL_DB_NAME;
this.isRestore = isRestore;
}

public String getDbName() {
return dbName;
}
Expand Down
80 changes: 78 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ public class RestoreStmt extends AbstractBackupStmt implements NotFallbackInPars
public static final String PROP_CLEAN_TABLES = "clean_tables";
public static final String PROP_CLEAN_PARTITIONS = "clean_partitions";
public static final String PROP_ATOMIC_RESTORE = "atomic_restore";
public static final String PROP_RESERVE_PRIVILEGE = "reserve_privilege";
public static final String PROP_RESERVE_CATALOG = "reserve_catalog";
public static final String PROP_RESERVE_WORKLOAD_GROUP = "reserve_workload_group";

private boolean allowLoad = false;
private ReplicaAllocation replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
Expand All @@ -58,15 +61,23 @@ public class RestoreStmt extends AbstractBackupStmt implements NotFallbackInPars
private boolean isAtomicRestore = false;
private byte[] meta = null;
private byte[] jobInfo = null;
private boolean reservePrivilege = false;
private boolean reserveCatalog = false;
private boolean reserveWorkloadGroup = false;

public RestoreStmt(LabelName labelName, String repoName, AbstractBackupTableRefClause restoreTableRefClause,
Map<String, String> properties) {
super(labelName, repoName, restoreTableRefClause, properties);
super(labelName, repoName, restoreTableRefClause, properties, false);
}

public RestoreStmt(LabelName labelName, String repoName,
Map<String, String> properties) {
super(labelName, repoName, null, properties, true);
}

public RestoreStmt(LabelName labelName, String repoName, AbstractBackupTableRefClause restoreTableRefClause,
Map<String, String> properties, byte[] meta, byte[] jobInfo) {
super(labelName, repoName, restoreTableRefClause, properties);
super(labelName, repoName, restoreTableRefClause, properties, false);
this.meta = meta;
this.jobInfo = jobInfo;
}
Expand Down Expand Up @@ -127,6 +138,18 @@ public boolean isAtomicRestore() {
return isAtomicRestore;
}

public boolean reservePrivilege() {
return reservePrivilege;
}

public boolean reserveCatalog() {
return reserveCatalog;
}

public boolean reserveWorkloadGroup() {
return reserveWorkloadGroup;
}

@Override
public void analyze(Analyzer analyzer) throws UserException {
if (repoName.equals(Repository.KEEP_ON_LOCAL_REPO_NAME)) {
Expand Down Expand Up @@ -212,6 +235,59 @@ public void analyzeProperties() throws AnalysisException {
// is atomic restore
isAtomicRestore = eatBooleanProperty(copiedProperties, PROP_ATOMIC_RESTORE, isAtomicRestore);

// reserve privilege
if (copiedProperties.containsKey(PROP_RESERVE_PRIVILEGE)) {
if (copiedProperties.get(PROP_RESERVE_PRIVILEGE).equalsIgnoreCase("true")) {
reservePrivilege = true;
} else if (copiedProperties.get(PROP_RESERVE_PRIVILEGE).equalsIgnoreCase("false")) {
reservePrivilege = false;
} else {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Invalid reserve_privilege value: " + copiedProperties.get(PROP_RESERVE_PRIVILEGE));
}
copiedProperties.remove(PROP_RESERVE_PRIVILEGE);
}

// reserve catalog
if (copiedProperties.containsKey(PROP_RESERVE_CATALOG)) {
if (copiedProperties.get(PROP_RESERVE_CATALOG).equalsIgnoreCase("true")) {
reserveCatalog = true;
} else if (copiedProperties.get(PROP_RESERVE_CATALOG).equalsIgnoreCase("false")) {
reserveCatalog = false;
} else {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Invalid reserve_catalog value: " + copiedProperties.get(PROP_RESERVE_CATALOG));
}
copiedProperties.remove(PROP_RESERVE_CATALOG);
}

// reserve workload group
if (copiedProperties.containsKey(PROP_RESERVE_WORKLOAD_GROUP)) {
if (copiedProperties.get(PROP_RESERVE_WORKLOAD_GROUP).equalsIgnoreCase("true")) {
reserveWorkloadGroup = true;
} else if (copiedProperties.get(PROP_RESERVE_WORKLOAD_GROUP).equalsIgnoreCase("false")) {
reserveWorkloadGroup = false;
} else {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Invalid reserve_workload_group value: " + copiedProperties.get(PROP_RESERVE_WORKLOAD_GROUP));
}
copiedProperties.remove(PROP_RESERVE_WORKLOAD_GROUP);
}

if (isBackupGlobal()) {
if (!properties.containsKey(PROP_RESERVE_PRIVILEGE)
&& !properties.containsKey(PROP_RESERVE_CATALOG)
&& !properties.containsKey(PROP_RESERVE_WORKLOAD_GROUP)) {
reservePrivilege = true;
reserveCatalog = true;
reserveWorkloadGroup = true;
}
} else {
reservePrivilege = false;
reserveCatalog = false;
reserveWorkloadGroup = false;
}

if (!copiedProperties.isEmpty()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Unknown restore job properties: " + copiedProperties.keySet());
Expand Down
Loading

0 comments on commit 8e57386

Please sign in to comment.