I'm not sure what I was doing wrong before or why I didn't think it would work (I'm still very new to drawables & selectors), but here it is now and it works:
IncDecButtonBackground.xml:
<?xml version="1.0" encoding="utf-8" ?>
<layer-list xmlns:android="http://schemas.android.com/apk/res/android">
<item>
<shape android:shape="rectangle">
<solid android:color="@color/Silver"/>
<stroke android:width="2dp" android:color="@color/Black"/>
<corners android:radius="0dp"/>
</shape>
</item>
<item android:gravity="fill" android:left="10dp" android:top="10dp" android:right="10dp" android:bottom="10dp">
<vector android:width="30dp" android:height="30dp" android:viewportWidth="30" android:viewportHeight="30">
<path android:pathData="M0 0L30 0L15 30Z" android:fillColor="@drawable/incdecbuttonarrowcolorselector"/>
</vector>
</item>
</layer-list>
IncDecButtonArrowColorSelector.xml:
<?xml version="1.0" encoding="utf-8" ?>
<selector xmlns:android="http://schemas.android.com/apk/res/android">
<item android:state_enabled="true" android:color="@Color /Black"/>
<item android:state_enabled="false" android:color="@Color /Gray"/>
</selector>