Append to MacPorts CFLAGS and CXXFLAGS rather than overwriting them.
--- Makefile.orig	2019-01-08 17:13:16.000000000 -0600
+++ Makefile	2019-01-08 17:32:37.000000000 -0600
@@ -7,7 +7,7 @@
 X_BUILD_IMPLICIT=1
 
 # -s silcently corrupts binaries on OS X, sigh -ReneR
-CFLAGS := -Wall -O2 # -O1 -ggdb # -fsanitize=address -fsanitize=undefined
+COMPILER_FLAGS := -Wall
 
 # for config.h
 CPPFLAGS += -I .
@@ -19,12 +19,13 @@
 cc-option = $(shell if $(CC) $(1) -S -o /dev/null -xc /dev/null \
             > /dev/null 2>&1; then echo "$(1)"; else echo "$(2)"; fi ;)
 
-CFLAGS += -funroll-loops -fomit-frame-pointer
+COMPILER_FLAGS += -funroll-loops -fomit-frame-pointer
 
 # we have some unimplemented colorspaces in the Image::iterator :-(
-CFLAGS += $(call cc-option,-Wno-switch -Wno-switch-enum,)
+COMPILER_FLAGS += $(call cc-option,-Wno-switch -Wno-switch-enum,)
 
-CXXFLAGS := $(CFLAGS) -Wno-sign-compare
+CFLAGS += $(COMPILER_FLAGS)
+CXXFLAGS += $(COMPILER_FLAGS) -Wno-sign-compare
 
 ifeq "$(STATIC)" "1"
 X_EXEFLAGS += -static