Skip to content

Commit 5cc4268

Browse files
tburchtburch
committed
add blob migration
1 parent 17e5256 commit 5cc4268

14 files changed

+426
-111
lines changed

‎Procfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
web: java $JAVA_OPTS -Ddw.mongo.type=uri -Ddw.mongo.uri=$MONGOHQ_URL -Ddw.server.connector.type=http -Ddw.server.connector.port=$PORT -Ddw.blobManager.deleteEnabled=$BLOB_DELETE_ENABLE -Ddw.blobManager.blobAccessTtl=$BLOB_TTL -jar target/jsonblob.jar server target/config/jsonblob.yml
1+
web: java $JAVA_OPTS -Ddw.mongo.type=uri -Ddw.mongo.uri=$MONGOHQ_URL -Ddw.server.connector.type=http -Ddw.server.connector.port=$PORT -Ddw.blobManager.fileSystemBlogDataDirectory=/var/jsonblob/data -Ddw.blobManager.blobAccessTtl=$BLOB_TTL -jar target/jsonblob.jar server target/config/jsonblob.yml

‎pom.xml

+7-2
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,20 @@
6363
<version>${io.dropwizard.version}</version>
6464
<scope>test</scope>
6565
</dependency>
66+
<dependency>
67+
<groupId>com.fasterxml.uuid</groupId>
68+
<artifactId>java-uuid-generator</artifactId>
69+
<version>3.1.4</version>
70+
</dependency>
6671
<dependency>
6772
<groupId>org.projectlombok</groupId>
6873
<artifactId>lombok</artifactId>
6974
<version>1.12.6</version>
7075
</dependency>
7176
<dependency>
7277
<groupId>org.mongodb</groupId>
73-
<artifactId>mongo-java-driver</artifactId>
74-
<version>2.14.3</version>
78+
<artifactId>mongodb-driver</artifactId>
79+
<version>3.3.0</version>
7580
</dependency>
7681
<dependency>
7782
<groupId>org.apache.commons</groupId>

‎src/main/java/com/lowtuna/jsonblob/JsonBlobApplication.java

+17-7
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66
import com.lowtuna.dropwizard.extras.heroku.RequestIdFilter;
77
import com.lowtuna.dropwizard.extras.view.handlebars.ConfiguredHandlebarsViewBundle;
88
import com.lowtuna.jsonblob.config.JsonBlobConfiguration;
9-
import com.lowtuna.jsonblob.core.BlobManager;
9+
import com.lowtuna.jsonblob.core.BlobMigrationJob;
10+
import com.lowtuna.jsonblob.core.FileSystemJsonBlobManager;
11+
import com.lowtuna.jsonblob.core.MongoDbJsonBlobManager;
1012
import com.lowtuna.jsonblob.healthcheck.CreateDeleteBlobHealthCheck;
1113
import com.lowtuna.jsonblob.healthcheck.MongoHealthCheck;
1214
import com.lowtuna.jsonblob.resource.ApiResource;
@@ -28,6 +30,8 @@
2830
import org.joda.time.format.PeriodFormatterBuilder;
2931

3032
import javax.ws.rs.core.MultivaluedMap;
33+
import java.util.concurrent.ScheduledExecutorService;
34+
import java.util.concurrent.TimeUnit;
3135

3236
@Slf4j
3337
public class JsonBlobApplication extends Application<JsonBlobConfiguration> {
@@ -82,22 +86,26 @@ public String getValue() {
8286

8387
DB mongoDBInstance = configuration.getMongoDbConfig().instance();
8488

85-
BlobManager blobManager = new BlobManager(
89+
ScheduledExecutorService scheduledExecutorService = configuration.getBlobManagerConfig().getScheduledExecutorService().instance(environment);
90+
91+
MongoDbJsonBlobManager mongoDbBlobManager = new MongoDbJsonBlobManager(
8692
mongoDBInstance,
8793
configuration.getBlobManagerConfig().getBlobCollectionName(),
88-
configuration.getBlobManagerConfig().getScheduledExecutorService().instance(environment),
94+
scheduledExecutorService,
8995
configuration.getBlobManagerConfig().getBlobCleanupFrequency(),
9096
configuration.getBlobManagerConfig().getBlobAccessTtl(),
9197
environment.metrics(),
9298
configuration.getBlobManagerConfig().isDeleteEnabled()
9399
);
94-
environment.lifecycle().manage(blobManager);
100+
environment.lifecycle().manage(mongoDbBlobManager);
101+
102+
FileSystemJsonBlobManager fileSystemBlobManager = new FileSystemJsonBlobManager(configuration.getBlobManagerConfig().getFileSystemBlogDataDirectory());
95103

96104
environment.healthChecks().register("MongoDB", new MongoHealthCheck(mongoDBInstance));
97-
environment.healthChecks().register("BlobManager", new CreateDeleteBlobHealthCheck(blobManager));
105+
environment.healthChecks().register("MongoDbJsonBlobManager", new CreateDeleteBlobHealthCheck(fileSystemBlobManager));
98106

99-
environment.jersey().register(new ApiResource(blobManager, configuration.getGoogleAnalyticsConfig()));
100-
environment.jersey().register(new JsonBlobEditorResource(blobManager, configuration.getGoogleAnalyticsConfig()));
107+
environment.jersey().register(new ApiResource(mongoDbBlobManager, fileSystemBlobManager, configuration.getGoogleAnalyticsConfig()));
108+
environment.jersey().register(new JsonBlobEditorResource(fileSystemBlobManager, mongoDbBlobManager, configuration.getGoogleAnalyticsConfig()));
101109
environment.jersey().getResourceConfig().getContainerResponseFilters().add(new GitTipHeaderFilter());
102110
environment.jersey().getResourceConfig().getContainerRequestFilters().add(new RequestIdFilter("X-Request-ID"));
103111

@@ -118,6 +126,8 @@ public ContainerResponse filter(ContainerRequest request, ContainerResponse resp
118126
return response;
119127
}
120128
});
129+
130+
scheduledExecutorService.scheduleWithFixedDelay(new BlobMigrationJob(mongoDbBlobManager, fileSystemBlobManager), 0, 1, TimeUnit.MINUTES);
121131
}
122132

123133
}

‎src/main/java/com/lowtuna/jsonblob/config/BlobManagerConfig.java

+5
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import org.hibernate.validator.constraints.NotEmpty;
99

1010
import javax.validation.constraints.NotNull;
11+
import java.io.File;
1112

1213
@Data
1314
@NoArgsConstructor
@@ -32,4 +33,8 @@ public class BlobManagerConfig {
3233
@NotNull
3334
private Duration blobAccessTtl = Duration.days(90);
3435

36+
@NotNull
37+
@JsonProperty("fileSystemBlogDataDirectory")
38+
private File fileSystemBlogDataDirectory;
39+
3540
}

‎src/main/java/com/lowtuna/jsonblob/core/BlobCleanupJob.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ public BlobCleanupJob(final DBCollection collection, Duration blobAccessTtl, Met
2424
this.collection = collection;
2525
this.blobAccessTtl = blobAccessTtl;
2626

27-
String[] attributes = new String[] { BlobManager.ACCESSED_ATTR_NAME, BlobManager.CREATED_ATTR_NAME, BlobManager.UPDATED_ATTR_NAME};
27+
String[] attributes = new String[] { MongoDbJsonBlobManager.ACCESSED_ATTR_NAME, MongoDbJsonBlobManager.CREATED_ATTR_NAME, MongoDbJsonBlobManager.UPDATED_ATTR_NAME};
2828
DecimalFormat periodFormat = new DecimalFormat("00");
2929

3030
int maxDays = blobAccessTtl.getUnit().equals(TimeUnit.DAYS) ? (int) blobAccessTtl.getQuantity() : 90;
@@ -50,7 +50,7 @@ protected Long loadValue() {
5050
public void run() {
5151
DateTime minLastAccessed = DateTime.now(DateTimeZone.UTC).minus(blobAccessTtl.toMilliseconds());
5252
BasicDBObject query = new BasicDBObject();
53-
query.put(BlobManager.ACCESSED_ATTR_NAME, BasicDBObjectBuilder.start("$lte", new Date(minLastAccessed.getMillis())).get());
53+
query.put(MongoDbJsonBlobManager.ACCESSED_ATTR_NAME, BasicDBObjectBuilder.start("$lte", new Date(minLastAccessed.getMillis())).get());
5454
log.info("removing all blobs that haven't been accessed since {}", minLastAccessed);
5555
WriteResult result = collection.remove(query);
5656
log.info("successfully removed {} blob(s)", result.getN());

‎src/main/java/com/lowtuna/jsonblob/core/BlobMigrationJob.java

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package com.lowtuna.jsonblob.core;
2+
3+
import com.google.common.base.Stopwatch;
4+
import com.mongodb.DBCursor;
5+
import com.mongodb.DBObject;
6+
import com.mongodb.MongoException;
7+
import lombok.extern.slf4j.Slf4j;
8+
import org.bson.types.ObjectId;
9+
10+
import java.util.concurrent.TimeUnit;
11+
12+
/**
13+
* Created by tburch on 11/15/16.
14+
*/
15+
@Slf4j
16+
public class BlobMigrationJob implements Runnable {
17+
18+
private final MongoDbJsonBlobManager mongoDbJsonBlobManager;
19+
private final FileSystemJsonBlobManager fileSystemJsonBlobManager;
20+
21+
public BlobMigrationJob(MongoDbJsonBlobManager mongoDbJsonBlobManager, FileSystemJsonBlobManager fileSystemJsonBlobManager) {
22+
this.mongoDbJsonBlobManager = mongoDbJsonBlobManager;
23+
this.fileSystemJsonBlobManager = fileSystemJsonBlobManager;
24+
}
25+
26+
@Override
27+
public void run() {
28+
Stopwatch stopwatch = new Stopwatch();
29+
stopwatch.start();
30+
int migratedBlobs = 0;
31+
32+
log.info("Starting blob migration");
33+
34+
DBCursor curs = mongoDbJsonBlobManager.getCollection().find().limit(5000);
35+
try {
36+
while (curs.hasNext()) {
37+
try {
38+
if (curs.hasNext()) {
39+
migratedBlobs++;
40+
DBObject o = curs.next();
41+
42+
ObjectId id = (ObjectId) o.get(MongoDbJsonBlobManager.ID_ATTR_NAME);
43+
try {
44+
String json = o.get(MongoDbJsonBlobManager.BLOB_ATTR_NAME).toString();
45+
46+
log.trace("Migrating blob {}", id);
47+
fileSystemJsonBlobManager.createBlob(json, id.toString());
48+
log.trace("Completed migrating blob {}", id);
49+
} finally {
50+
// mongoDbJsonBlobManager.deleteBlob(id.toString());
51+
}
52+
53+
if (migratedBlobs % 100 == 0) {
54+
log.info("Migrated {} blobs...", migratedBlobs);
55+
}
56+
}
57+
} catch (MongoException e) {
58+
log.warn("Error while migrating blob", e);
59+
} catch (Exception e) {
60+
log.warn("Caught exception while migrating blob", e);
61+
}
62+
}
63+
} catch (Exception e) {
64+
log.warn("Caught exception while migrating blobs", e);
65+
}
66+
log.info("Completed migrating {} blobs in {}ms", migratedBlobs, stopwatch.elapsed(TimeUnit.MILLISECONDS));
67+
}
68+
}

‎src/main/java/com/lowtuna/jsonblob/core/BlobNotFoundException.java

+6-7
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,31 @@
11
package com.lowtuna.jsonblob.core;
22

33
import lombok.Getter;
4-
import org.bson.types.ObjectId;
54

65
@Getter
76
public class BlobNotFoundException extends Exception {
8-
private final ObjectId id;
7+
private final String id;
98

10-
public BlobNotFoundException(ObjectId id) {
9+
public BlobNotFoundException(String id) {
1110
this.id = id;
1211
}
1312

14-
public BlobNotFoundException(String message, ObjectId id) {
13+
public BlobNotFoundException(String message, String id) {
1514
super(message);
1615
this.id = id;
1716
}
1817

19-
public BlobNotFoundException(String message, Throwable cause, ObjectId id) {
18+
public BlobNotFoundException(String message, Throwable cause, String id) {
2019
super(message, cause);
2120
this.id = id;
2221
}
2322

24-
public BlobNotFoundException(Throwable cause, ObjectId id) {
23+
public BlobNotFoundException(Throwable cause, String id) {
2524
super(cause);
2625
this.id = id;
2726
}
2827

29-
public BlobNotFoundException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace, ObjectId id) {
28+
public BlobNotFoundException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace, String id) {
3029
super(message, cause, enableSuppression, writableStackTrace);
3130
this.id = id;
3231
}

0 commit comments

Comments
 (0)