Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement AutoRecord for new snapshots. #32

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Implement AutoRecord for new snapshots. #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion FBSnapshotTestCase/FBSnapshotTestCase.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,12 @@ NS_ASSUME_NONNULL_BEGIN
*/
@property (readwrite, nonatomic, assign) BOOL recordMode;

/**
When YES, a test will run and fail when no reference image exists, without needing to run
recordMode first. The fail image is stored and can be reviewed and accepted as a reference.
*/
@property (readwrite, nonatomic, assign) BOOL autoRecord;

/**
When set, allows fine-grained control over what you want the file names to include.

Expand All @@ -147,7 +153,6 @@ NS_ASSUME_NONNULL_BEGIN

self.fileNameOptions = (FBSnapshotTestCaseFileNameIncludeOptionDevice | FBSnapshotTestCaseFileNameIncludeOptionOS);
*/

@property (readwrite, nonatomic, assign) FBSnapshotTestCaseFileNameIncludeOption fileNameOptions;

/**
Expand Down Expand Up @@ -207,6 +212,7 @@ NS_ASSUME_NONNULL_BEGIN
defaultReferenceDirectory:(nullable NSString *)defaultReferenceDirectory
defaultImageDiffDirectory:(nullable NSString *)defaultImageDiffDirectory;


/**
Performs the comparison or records a snapshot of the layer if recordMode is YES.
@param layer The Layer to snapshot.
Expand Down
65 changes: 55 additions & 10 deletions FBSnapshotTestCase/FBSnapshotTestCase.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,17 @@ - (void)setRecordMode:(BOOL)recordMode
_snapshotController.recordMode = recordMode;
}

- (BOOL)autoRecord
{
return _snapshotController.autoRecord;
}

- (void)setAutoRecord:(BOOL)autoRecord
{
NSAssert1(_snapshotController, @"%s cannot be called before [super setUp]", __FUNCTION__);
_snapshotController.autoRecord = autoRecord;
}

- (FBSnapshotTestCaseFileNameIncludeOption)fileNameOptions
{
return _snapshotController.fileNameOptions;
Expand Down Expand Up @@ -120,16 +131,12 @@ - (NSString *)snapshotVerifyViewOrLayer:(id)viewOrLayer
NSMutableArray *errors = [NSMutableArray array];

if (self.recordMode) {
NSString *referenceImagesDirectory = [NSString stringWithFormat:@"%@%@", referenceImageDirectory, suffixes.firstObject];
BOOL referenceImageSaved = [self _compareSnapshotOfViewOrLayer:viewOrLayer referenceImagesDirectory:referenceImagesDirectory imageDiffDirectory:imageDiffDirectory identifier:(identifier) perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance error:&error];
if (!referenceImageSaved) {
[errors addObject:error];
}
[self _writeReferenceImageOfViewOrLayer:viewOrLayer suffixes:suffixes referenceImageDirectory:referenceImageDirectory identifier:identifier perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance errors:errors];
} else {
for (NSString *suffix in suffixes) {
NSString *referenceImagesDirectory = [NSString stringWithFormat:@"%@%@", referenceImageDirectory, suffix];
BOOL referenceImageAvailable = [self referenceImageRecordedInDirectory:referenceImagesDirectory identifier:(identifier) error:&error];

if (referenceImageAvailable) {
BOOL comparisonSuccess = [self _compareSnapshotOfViewOrLayer:viewOrLayer referenceImagesDirectory:referenceImagesDirectory imageDiffDirectory:imageDiffDirectory identifier:identifier perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance error:&error];
[errors removeAllObjects];
Expand All @@ -139,19 +146,24 @@ - (NSString *)snapshotVerifyViewOrLayer:(id)viewOrLayer
} else {
[errors addObject:error];
}
} else if (!referenceImageAvailable && self.autoRecord) {
[self _writeReferenceImageOfViewOrLayer:viewOrLayer suffixes:suffixes referenceImageDirectory:referenceImageDirectory identifier:identifier perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance errors:errors];
} else {
[errors addObject:error];
}
}
}

if (!testSuccess) {
return [NSString stringWithFormat:@"Snapshot comparison failed: %@", errors.firstObject];
}
if (self.recordMode) {
return @"Test ran in record mode. Reference image is now saved. Disable record mode to perform an actual snapshot comparison!";
}

else if (!testSuccess && !self.autoRecord) {
return [NSString stringWithFormat:@"Snapshot comparison failed: %@", errors.firstObject];
}
else if (!testSuccess && self.autoRecord) {
return [NSString stringWithFormat:@"No previous reference image. New image has been stored for approval."];
}

return nil;
}

Expand Down Expand Up @@ -260,6 +272,22 @@ - (NSString *)getImageDiffDirectoryWithDefault:(NSString *)dir

#pragma mark - Private API

- (BOOL)_compareSnapshotOfViewOrLayer:(id)viewOrLayer
referenceImagesDirectory:(NSString *)referenceImagesDirectory
imageDiffDirectory:(NSString *)imageDiffDirectory
identifier:(NSString *)identifier
overallTolerance:(CGFloat)overallTolerance
error:(NSError **)errorPtr
{
return [self _compareSnapshotOfViewOrLayer:(id)viewOrLayer
referenceImagesDirectory:referenceImagesDirectory
imageDiffDirectory:imageDiffDirectory
identifier:identifier
perPixelTolerance:0
overallTolerance:overallTolerance
error:errorPtr];
}

- (BOOL)_compareSnapshotOfViewOrLayer:(id)viewOrLayer
referenceImagesDirectory:(NSString *)referenceImagesDirectory
imageDiffDirectory:(NSString *)imageDiffDirectory
Expand All @@ -278,4 +306,21 @@ - (BOOL)_compareSnapshotOfViewOrLayer:(id)viewOrLayer
error:errorPtr];
}

- (void)_writeReferenceImageOfViewOrLayer:(id)viewOrLayer
suffixes:(NSOrderedSet *)suffixes
referenceImageDirectory:(NSString *)referenceImageDirectory
identifier:(NSString *)identifier
perPixelTolerance:(CGFloat)perPixelTolerance
overallTolerance:(CGFloat)overallTolerance
errors:(NSMutableArray *)errors
{
NSError *error = nil;
NSString *referenceImagesDirectory = [NSString stringWithFormat:@"%@%@", referenceImageDirectory, suffixes.firstObject];
NSString *imageDiffDirectory = [self getImageDiffDirectoryWithDefault:nil];
BOOL referenceImageSaved = [self _compareSnapshotOfViewOrLayer:viewOrLayer referenceImagesDirectory:referenceImagesDirectory imageDiffDirectory:imageDiffDirectory identifier:(identifier) perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance error:&error];
if (!referenceImageSaved) {
[errors addObject:error];
}
}

@end
5 changes: 5 additions & 0 deletions FBSnapshotTestCase/FBSnapshotTestController.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ extern NSString *const FBDiffedImageKey;
*/
@interface FBSnapshotTestController : NSObject

/**
Auto record snapshots on first run of new tests.
*/
@property (readwrite, nonatomic, assign) BOOL autoRecord;

/**
Record snapshots.
*/
Expand Down
94 changes: 52 additions & 42 deletions FBSnapshotTestCase/FBSnapshotTestController.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ - (instancetype)initWithTestClass:(Class)testClass;
if (self = [super init]) {
_folderName = NSStringFromClass(testClass);
_fileNameOptions = FBSnapshotTestCaseFileNameIncludeOptionScreenScale;

_fileManager = [[NSFileManager alloc] init];
}
return self;
Expand Down Expand Up @@ -102,8 +102,13 @@ - (BOOL)compareSnapshotOfViewOrLayer:(id)viewOrLayer
overallTolerance:(CGFloat)overallTolerance
error:(NSError **)errorPtr
{
UIImage *referenceImage = [self referenceImageForSelector:selector identifier:identifier error:errorPtr];
BOOL noReferenceImage = (referenceImage == nil);

if (self.recordMode) {
return [self _recordSnapshotOfViewOrLayer:viewOrLayer selector:selector identifier:identifier error:errorPtr];
} else if (self.autoRecord && noReferenceImage) {
return [self _recordSnapshotOfViewOrLayer:viewOrLayer selector:selector identifier:identifier error:errorPtr];
} else {
return [self _performPixelComparisonWithViewOrLayer:viewOrLayer selector:selector identifier:identifier perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance error:errorPtr];
}
Expand All @@ -117,14 +122,22 @@ - (UIImage *)referenceImageForSelector:(SEL)selector
UIImage *image = [UIImage imageWithContentsOfFile:filePath];
if (nil == image && NULL != errorPtr) {
BOOL exists = [_fileManager fileExistsAtPath:filePath];
if (!exists) {
if (!exists && !self.autoRecord) {
*errorPtr = [NSError errorWithDomain:FBSnapshotTestControllerErrorDomain
code:FBSnapshotTestControllerErrorCodeNeedsRecord
userInfo:@{
FBReferenceImageFilePathKey: filePath,
NSLocalizedDescriptionKey: @"Unable to load reference image.",
NSLocalizedFailureReasonErrorKey: @"Reference image not found. You need to run the test in record mode",
}];
} else if (!exists && self.autoRecord) {
*errorPtr = [NSError errorWithDomain:FBSnapshotTestControllerErrorDomain
code:FBSnapshotTestControllerErrorCodeNeedsRecord
userInfo:@{
FBReferenceImageFilePathKey : filePath,
NSLocalizedDescriptionKey : @"Unable to load reference image.",
NSLocalizedFailureReasonErrorKey : @"Reference image not found. You need to run the test in record mode",
}];
FBReferenceImageFilePathKey: filePath,
NSLocalizedDescriptionKey: @"Unable to load reference image.",
NSLocalizedFailureReasonErrorKey: @"Reference image not found. Auto-recorded image saved for review",
}];
} else {
*errorPtr = [NSError errorWithDomain:FBSnapshotTestControllerErrorDomain
code:FBSnapshotTestControllerErrorCodeUnknown
Expand Down Expand Up @@ -156,21 +169,21 @@ - (BOOL)compareReferenceImage:(UIImage *)referenceImage
if (sameImageDimensions && [referenceImage fb_compareWithImage:image perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance]) {
return YES;
}

if (NULL != errorPtr) {
NSString *errorDescription = sameImageDimensions ? @"Images different" : @"Images different sizes";
NSString *errorReason = sameImageDimensions ? [NSString stringWithFormat:@"image pixels differed by more than %.2f%% from the reference image", overallTolerance * 100] : [NSString stringWithFormat:@"referenceImage:%@, image:%@", NSStringFromCGSize(referenceImage.size), NSStringFromCGSize(image.size)];
FBSnapshotTestControllerErrorCode errorCode = sameImageDimensions ? FBSnapshotTestControllerErrorCodeImagesDifferent : FBSnapshotTestControllerErrorCodeImagesDifferentSizes;

*errorPtr = [NSError errorWithDomain:FBSnapshotTestControllerErrorDomain
code:errorCode
userInfo:@{
NSLocalizedDescriptionKey : errorDescription,
NSLocalizedFailureReasonErrorKey : errorReason,
FBReferenceImageKey : referenceImage,
FBCapturedImageKey : image,
FBDiffedImageKey : [referenceImage fb_diffWithImage:image],
}];
NSLocalizedDescriptionKey : errorDescription,
NSLocalizedFailureReasonErrorKey : errorReason,
FBReferenceImageKey : referenceImage,
FBCapturedImageKey : image,
FBDiffedImageKey : [referenceImage fb_diffWithImage:image],
}];
}
return NO;
}
Expand All @@ -183,11 +196,11 @@ - (BOOL)saveFailedReferenceImage:(UIImage *)referenceImage
{
NSData *referencePNGData = UIImagePNGRepresentation(referenceImage);
NSData *testPNGData = UIImagePNGRepresentation(testImage);

NSString *referencePath = [self _failedFilePathForSelector:selector
identifier:identifier
fileNameType:FBTestSnapshotFileNameTypeFailedReference];

NSError *creationError = nil;
BOOL didCreateDir = [_fileManager createDirectoryAtPath:[referencePath stringByDeletingLastPathComponent]
withIntermediateDirectories:YES
Expand All @@ -199,34 +212,33 @@ - (BOOL)saveFailedReferenceImage:(UIImage *)referenceImage
}
return NO;
}

if (![referencePNGData writeToFile:referencePath options:NSDataWritingAtomic error:errorPtr]) {
if (![referencePNGData writeToFile:referencePath options:NSDataWritingAtomic error:errorPtr] && !self.autoRecord) {
return NO;
}

NSString *testPath = [self _failedFilePathForSelector:selector
identifier:identifier
fileNameType:FBTestSnapshotFileNameTypeFailedTest];

if (![testPNGData writeToFile:testPath options:NSDataWritingAtomic error:errorPtr]) {
return NO;
}

NSString *diffPath = [self _failedFilePathForSelector:selector
identifier:identifier
fileNameType:FBTestSnapshotFileNameTypeFailedTestDiff];

UIImage *diffImage = [referenceImage fb_diffWithImage:testImage];
NSData *diffImageData = UIImagePNGRepresentation(diffImage);

if (![diffImageData writeToFile:diffPath options:NSDataWritingAtomic error:errorPtr]) {
if (![diffImageData writeToFile:diffPath options:NSDataWritingAtomic error:errorPtr] && !self.autoRecord) {
return NO;
}

NSLog(@"If you have Kaleidoscope installed you can run this command to see an image diff:\n"
@"ksdiff \"%@\" \"%@\"",
referencePath, testPath);

return YES;
}

Expand Down Expand Up @@ -255,12 +267,12 @@ - (NSString *)_fileNameForSelector:(SEL)selector
if (0 < identifier.length) {
fileName = [fileName stringByAppendingFormat:@"_%@", identifier];
}

BOOL noFileNameOption = (self.fileNameOptions & FBSnapshotTestCaseFileNameIncludeOptionNone) == FBSnapshotTestCaseFileNameIncludeOptionNone;
if (!noFileNameOption) {
fileName = FBFileNameIncludeNormalizedFileNameFromOption(fileName, self.fileNameOptions);
fileName = FBFileNameIncludeNormalizedFileNameFromOption(fileName, self.fileNameOptions);
}

fileName = [fileName stringByAppendingPathExtension:@"png"];
return fileName;
}
Expand All @@ -283,7 +295,7 @@ - (NSString *)_failedFilePathForSelector:(SEL)selector
NSString *fileName = [self _fileNameForSelector:selector
identifier:identifier
fileNameType:fileNameType];

NSString *filePath = [_imageDiffDirectory stringByAppendingPathComponent:self.folderName];
filePath = [filePath stringByAppendingPathComponent:fileName];
return filePath;
Expand All @@ -297,18 +309,16 @@ - (BOOL)_performPixelComparisonWithViewOrLayer:(id)viewOrLayer
error:(NSError **)errorPtr
{
UIImage *referenceImage = [self referenceImageForSelector:selector identifier:identifier error:errorPtr];
if (nil != referenceImage) {
UIImage *snapshot = [self _imageForViewOrLayer:viewOrLayer];
BOOL imagesSame = [self compareReferenceImage:referenceImage toImage:snapshot perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance error:errorPtr];
if (!imagesSame) {
NSError *saveError = nil;
if ([self saveFailedReferenceImage:referenceImage testImage:snapshot selector:selector identifier:identifier error:&saveError] == NO) {
NSLog(@"Error saving test images: %@", saveError);
}
UIImage *snapshot = [self _imageForViewOrLayer:viewOrLayer];
BOOL imagesSame = [self compareReferenceImage:referenceImage toImage:snapshot perPixelTolerance:perPixelTolerance overallTolerance:overallTolerance error:errorPtr];

if (!imagesSame) {
NSError *saveError = nil;
if ([self saveFailedReferenceImage:referenceImage testImage:snapshot selector:selector identifier:identifier error:&saveError] == NO) {
NSLog(@"Error saving test images: %@", saveError);
}
return imagesSame;
}
return NO;
return imagesSame;
}

- (BOOL)_recordSnapshotOfViewOrLayer:(id)viewOrLayer
Expand Down Expand Up @@ -350,8 +360,8 @@ - (BOOL)_saveReferenceImage:(UIImage *)image
*errorPtr = [NSError errorWithDomain:FBSnapshotTestControllerErrorDomain
code:FBSnapshotTestControllerErrorCodePNGCreationFailed
userInfo:@{
FBReferenceImageFilePathKey : filePath,
}];
FBReferenceImageFilePathKey : filePath,
}];
}
}
}
Expand Down
Loading